[ { "path": ".dockerignore", "content": "# Git\n.git/\n.gitignore\n\n# GitHub\n.github/\n\n# Byte-compiled / optimized / DLL files\n**/__pycache__\n**/*.py[cod]\n\n# Caches\n.mypy_cache/\n.pytest_cache/\n.ruff_cache/\n\n# Distribution / packaging\nbuild/\ndist/\n*.egg-info*\n\n# Virtual environment\n.env\n.venv/\nvenv/\n\n# IntelliJ IDEA\n.idea/\n\n# Visual Studio\n.vscode/\n\n# Test and development files\ntest-datastore/\ntests/\n*.md\n!README.md\n\n# Temporary and log files\n*.log\n*.tmp\ntmp/\ntemp/\n\n# Training data and large files\ntrain-data/\nworks-data/\n\n# Container files\nDockerfile*\ndocker-compose*.yml\n.dockerignore\n\n# Development certificates and keys\n*.pem\n*.key\n*.crt\nprofile_output.prof\n\n# Large binary files that shouldn't be in container\n*.pdf\nchrome.json" }, { "path": ".github/FUNDING.yml", "content": "# These are supported funding model platforms\n\ngithub: dgtlmoon\n" }, { "path": ".github/ISSUE_TEMPLATE/bug_report.md", "content": "---\nname: Bug report\nabout: Create a bug report, if you don't follow this template, your report will be DELETED\ntitle: ''\nlabels: 'triage'\nassignees: 'dgtlmoon'\n\n---\n\n**DO NOT USE THIS FORM TO REPORT THAT A PARTICULAR WEBSITE IS NOT SCRAPING/WATCHING AS EXPECTED**\n\nThis form is only for direct bugs and feature requests todo directly with the software.\n\nPlease report watched websites (full URL and _any_ settings) that do not work with changedetection.io as expected [**IN THE DISCUSSION FORUMS**](https://github.com/dgtlmoon/changedetection.io/discussions) or your report will be deleted\n\nCONSIDER TAKING OUT A SUBSCRIPTION FOR A SMALL PRICE PER MONTH, YOU GET THE BENEFIT OF USING OUR PAID PROXIES AND FURTHERING THE DEVELOPMENT OF CHANGEDETECTION.IO\n\nTHANK YOU\n\n\n\n\n\n**Describe the bug**\nA clear and concise description of what the bug is.\n\n**Version**\n*Exact version* in the top right area: 0....\n\n**How did you install?**\n\nDocker, Pip, from source directly etc\n\n**To Reproduce**\n\nSteps to reproduce the behavior:\n1. Go to '...'\n2. Click on '....'\n3. Scroll down to '....'\n4. See error\n\n! ALWAYS INCLUDE AN EXAMPLE URL WHERE IT IS POSSIBLE TO RE-CREATE THE ISSUE - USE THE 'SHARE WATCH' FEATURE AND PASTE IN THE SHARE-LINK!\n\n**Expected behavior**\nA clear and concise description of what you expected to happen.\n\n**Screenshots**\nIf applicable, add screenshots to help explain your problem.\n\n**Desktop (please complete the following information):**\n - OS: [e.g. iOS] \n - Browser [e.g. chrome, safari]\n - Version [e.g. 22]\n\n**Smartphone (please complete the following information):**\n - Device: [e.g. iPhone6]\n - OS: [e.g. iOS8.1]\n - Browser [e.g. stock browser, safari]\n - Version [e.g. 22]\n\n**Additional context**\nAdd any other context about the problem here.\n" }, { "path": ".github/ISSUE_TEMPLATE/feature_request.md", "content": "---\nname: Feature request\nabout: Suggest an idea for this project\ntitle: '[feature]'\nlabels: 'enhancement'\nassignees: ''\n\n---\n**Version and OS**\nFor example, 0.123 on linux/docker\n\n**Is your feature request related to a problem? Please describe.**\nA clear and concise description of what the problem is. Ex. I'm always frustrated when [...]\n\n**Describe the solution you'd like**\nA clear and concise description of what you want to happen.\n\n**Describe the use-case and give concrete real-world examples**\nAttach any HTML/JSON, give links to sites, screenshots etc, we are not mind readers\n\n\n**Additional context**\nAdd any other context or screenshots about the feature request here.\n" }, { "path": ".github/actions/extract-memory-report/action.yml", "content": "name: 'Extract Memory Test Report'\ndescription: 'Extracts and displays memory test report from a container'\ninputs:\n container-name:\n description: 'Name of the container to extract logs from'\n required: true\n python-version:\n description: 'Python version for artifact naming'\n required: true\n output-dir:\n description: 'Directory to store output logs'\n required: false\n default: 'output-logs'\n\nruns:\n using: \"composite\"\n steps:\n - name: Create output directory\n shell: bash\n run: |\n mkdir -p ${{ inputs.output-dir }}\n\n - name: Dump container log\n shell: bash\n run: |\n echo \"Disabled for now\"\n# return\n# docker logs ${{ inputs.container-name }} > ${{ inputs.output-dir }}/${{ inputs.container-name }}-stdout-${{ inputs.python-version }}.txt 2>&1 || echo \"Could not get stdout\"\n# docker logs ${{ inputs.container-name }} 2> ${{ inputs.output-dir }}/${{ inputs.container-name }}-stderr-${{ inputs.python-version }}.txt || echo \"Could not get stderr\"\n\n - name: Extract and display memory test report\n shell: bash\n run: |\n echo \"Disabled for now\"\n# echo \"Extracting test-memory.log from container...\"\n# docker cp ${{ inputs.container-name }}:/app/changedetectionio/test-memory.log ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log || echo \"test-memory.log not found in container\"\n#\n# echo \"=== Top 10 Highest Peak Memory Tests ===\"\n# if [ -f ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log ]; then\n# grep \"Peak memory:\" ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log | \\\n# sed 's/.*Peak memory: //' | \\\n# paste -d'|' - <(grep \"Peak memory:\" ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log) | \\\n# sort -t'|' -k1 -nr | \\\n# cut -d'|' -f2 | \\\n# head -10\n# echo \"\"\n# echo \"=== Full Memory Test Report ===\"\n# cat ${{ inputs.output-dir }}/test-memory-${{ inputs.python-version }}.log\n# else\n# echo \"No memory log available\"\n# fi\n" }, { "path": ".github/dependabot.yml", "content": "version: 2\nupdates:\n - package-ecosystem: github-actions\n directory: /\n schedule:\n interval: \"weekly\"\n groups:\n all:\n patterns:\n - \"*\"\n - package-ecosystem: pip\n directory: /\n schedule:\n interval: \"weekly\"\n" }, { "path": ".github/nginx-reverse-proxy-test.conf", "content": "server {\n listen 80;\n server_name localhost;\n\n # Test basic reverse proxy to changedetection.io\n location / {\n proxy_pass http://changedet-app:5000;\n proxy_set_header Host $host;\n proxy_set_header X-Real-IP $remote_addr;\n proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n proxy_set_header X-Forwarded-Proto $scheme;\n\n # WebSocket support\n proxy_http_version 1.1;\n proxy_set_header Upgrade $http_upgrade;\n proxy_set_header Connection \"upgrade\";\n }\n\n # Test subpath deployment with X-Forwarded-Prefix\n location /changedet-sub/ {\n proxy_pass http://changedet-app:5000/;\n proxy_set_header X-Forwarded-Prefix /changedet-sub;\n proxy_set_header Host $host;\n proxy_set_header X-Real-IP $remote_addr;\n proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n proxy_set_header X-Forwarded-Proto $scheme;\n\n # WebSocket support\n proxy_http_version 1.1;\n proxy_set_header Upgrade $http_upgrade;\n proxy_set_header Connection \"upgrade\";\n }\n}\n" }, { "path": ".github/test/Dockerfile-alpine", "content": "# Taken from https://github.com/linuxserver/docker-changedetection.io/blob/main/Dockerfile\n# Test that we can still build on Alpine (musl modified libc https://musl.libc.org/)\n# Some packages wont install via pypi because they dont have a wheel available under this architecture.\n\nFROM ghcr.io/linuxserver/baseimage-alpine:3.22\nENV PYTHONUNBUFFERED=1\n\nCOPY requirements.txt /requirements.txt\n\nARG TARGETPLATFORM\n\nRUN \\\n apk add --update --no-cache --virtual=build-dependencies \\\n build-base \\\n cargo \\\n git \\\n jpeg-dev \\\n libc-dev \\\n libffi-dev \\\n libxslt-dev \\\n openssl-dev \\\n python3-dev \\\n file \\\n zip \\\n zlib-dev && \\\n apk add --update --no-cache \\\n libjpeg \\\n libxslt \\\n file \\\n nodejs \\\n poppler-utils \\\n python3 \\\n glib \\\n libsm \\\n libxext \\\n libxrender && \\\n case \"$TARGETPLATFORM\" in \\\n linux/arm/v7|linux/arm/v8) \\\n echo \"INFO: Skipping py3-opencv on $TARGETPLATFORM (using pixelmatch fallback)\" \\\n ;; \\\n *) \\\n apk add --update --no-cache py3-opencv || echo \"WARN: py3-opencv install failed, using pixelmatch fallback\" \\\n ;; \\\n esac && \\\n echo \"**** pip3 install test of changedetection.io ****\" && \\\n python3 -m venv /lsiopy && \\\n pip install -U pip wheel setuptools && \\\n pip install -U --no-cache-dir --find-links https://wheel-index.linuxserver.io/alpine-3.22/ -r /requirements.txt && \\\n apk del --purge \\\n build-dependencies\n" }, { "path": ".github/workflows/codeql-analysis.yml", "content": "# For most projects, this workflow file will not need changing; you simply need\n# to commit it to your repository.\n#\n# You may wish to alter this file to override the set of languages analyzed,\n# or to provide custom queries or build logic.\n#\n# ******** NOTE ********\n# We have attempted to detect the languages in your repository. Please check\n# the `language` matrix defined below to confirm you have the correct set of\n# supported CodeQL languages.\n#\nname: \"CodeQL\"\n\non:\n schedule:\n - cron: '27 9 * * 4'\n\njobs:\n analyze:\n name: Analyze\n runs-on: ubuntu-latest\n\n strategy:\n fail-fast: false\n matrix:\n language: [ 'javascript', 'python' ]\n # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]\n # Learn more:\n # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed\n\n steps:\n - name: Checkout repository\n uses: actions/checkout@v6\n\n # Initializes the CodeQL tools for scanning.\n - name: Initialize CodeQL\n uses: github/codeql-action/init@v4\n with:\n languages: ${{ matrix.language }}\n # If you wish to specify custom queries, you can do so here or in a config file.\n # By default, queries listed here will override any specified in a config file.\n # Prefix the list here with \"+\" to use these queries and those in the config file.\n # queries: ./path/to/local/query, your-org/your-repo/queries@main\n\n # Autobuild attempts to build any compiled languages (C/C++, C#, or Java).\n # If this step fails, then you should remove it and run the build manually (see below)\n - name: Autobuild\n uses: github/codeql-action/autobuild@v4\n\n # ℹ️ Command-line programs to run using the OS shell.\n # 📚 https://git.io/JvXDl\n\n # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines\n # and modify them (or add more) to build your code if your project\n # uses a compiled language\n\n #- run: |\n # make bootstrap\n # make release\n\n - name: Perform CodeQL Analysis\n uses: github/codeql-action/analyze@v4\n" }, { "path": ".github/workflows/containers.yml", "content": "name: Build and push containers\n\non:\n # Automatically triggered by a testing workflow passing, but this is only checked when it lands in the `master`/default branch\n# workflow_run:\n# workflows: [\"ChangeDetection.io Test\"]\n# branches: [master]\n# tags: ['0.*']\n# types: [completed]\n\n # Or a new tagged release\n release:\n types: [published, edited]\n\n push:\n branches:\n - master\n\njobs:\n metadata:\n runs-on: ubuntu-latest\n steps:\n - name: Show metadata\n run: |\n echo SHA ${{ github.sha }}\n echo github.ref: ${{ github.ref }}\n echo github_ref: $GITHUB_REF\n echo Event name: ${{ github.event_name }}\n echo Ref ${{ github.ref }}\n echo c: ${{ github.event.workflow_run.conclusion }}\n echo r: ${{ github.event.workflow_run }}\n echo tname: \"${{ github.event.release.tag_name }}\"\n echo headbranch: -${{ github.event.workflow_run.head_branch }}-\n set\n\n build-push-containers:\n runs-on: ubuntu-latest\n # If the testing workflow has a success, then we build to :latest\n # Or if we are in a tagged release scenario.\n if: ${{ github.event.workflow_run.conclusion == 'success' }} || ${{ github.event.release.tag_name }} != ''\n steps:\n - uses: actions/checkout@v6\n - name: Set up Python 3.11\n uses: actions/setup-python@v6\n with:\n python-version: 3.11\n\n - name: Cache pip packages\n uses: actions/cache@v5\n with:\n path: ~/.cache/pip\n key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}\n restore-keys: |\n ${{ runner.os }}-pip-\n\n - name: Install dependencies\n run: |\n python -m pip install --upgrade pip\n pip install flake8 pytest\n if [ -f requirements.txt ]; then pip install -r requirements.txt; fi\n\n - name: Create release metadata\n run: |\n # COPY'ed by Dockerfile into changedetectionio/ of the image, then read by the server in store.py\n echo ${{ github.sha }} > changedetectionio/source.txt\n echo ${{ github.ref }} > changedetectionio/tag.txt\n\n - name: Set up QEMU\n uses: docker/setup-qemu-action@v4\n with:\n image: tonistiigi/binfmt:latest\n platforms: all\n\n - name: Login to GitHub Container Registry\n uses: docker/login-action@v4\n with:\n registry: ghcr.io\n username: ${{ github.actor }}\n password: ${{ secrets.GITHUB_TOKEN }}\n\n - name: Login to Docker Hub Container Registry\n uses: docker/login-action@v4\n with:\n username: ${{ secrets.DOCKER_HUB_USERNAME }}\n password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}\n\n - name: Set up Docker Buildx\n id: buildx\n uses: docker/setup-buildx-action@v4\n with:\n install: true\n version: latest\n driver-opts: image=moby/buildkit:master\n\n # master branch -> :dev container tag\n - name: Docker meta :dev\n if: ${{ github.ref == 'refs/heads/master' && github.event_name != 'release' }}\n uses: docker/metadata-action@v6\n id: meta_dev\n with:\n images: |\n ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io\n ghcr.io/${{ github.repository }}\n tags: |\n type=raw,value=dev\n labels: |\n org.opencontainers.image.created=${{ github.event.release.published_at }}\n org.opencontainers.image.description=Website, webpage change detection, monitoring and notifications.\n org.opencontainers.image.documentation=https://changedetection.io\n org.opencontainers.image.revision=${{ github.sha }}\n org.opencontainers.image.source=https://github.com/dgtlmoon/changedetection.io\n org.opencontainers.image.title=changedetection.io\n org.opencontainers.image.url=https://changedetection.io\n\n - name: Build and push :dev\n id: docker_build\n if: ${{ github.ref == 'refs/heads/master' && github.event_name != 'release' }}\n uses: docker/build-push-action@v7\n with:\n context: ./\n file: ./Dockerfile\n push: true\n tags: ${{ steps.meta_dev.outputs.tags }}\n labels: ${{ steps.meta_dev.outputs.labels }}\n platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8\n cache-from: type=gha\n cache-to: type=gha,mode=max\n\n# Looks like this was disabled\n# provenance: false\n\n # A new tagged release is required, which builds :tag and :latest\n - name: Debug release info\n if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.')\n run: |\n echo \"Release tag: ${{ github.event.release.tag_name }}\"\n echo \"Github ref: ${{ github.ref }}\"\n echo \"Github ref name: ${{ github.ref_name }}\"\n\n - name: Docker meta :tag\n if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.')\n uses: docker/metadata-action@v6\n id: meta\n with:\n images: |\n ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io\n ghcr.io/dgtlmoon/changedetection.io\n tags: |\n type=semver,pattern={{version}},value=${{ github.event.release.tag_name }}\n type=semver,pattern={{major}}.{{minor}},value=${{ github.event.release.tag_name }}\n type=semver,pattern={{major}},value=${{ github.event.release.tag_name }}\n type=raw,value=latest\n labels: |\n org.opencontainers.image.created=${{ github.event.release.published_at }}\n org.opencontainers.image.description=Website, webpage change detection, monitoring and notifications.\n org.opencontainers.image.documentation=https://changedetection.io\n org.opencontainers.image.revision=${{ github.sha }}\n org.opencontainers.image.source=https://github.com/dgtlmoon/changedetection.io\n org.opencontainers.image.title=changedetection.io\n org.opencontainers.image.url=https://changedetection.io\n org.opencontainers.image.version=${{ github.event.release.tag_name }}\n\n - name: Build and push :tag\n id: docker_build_tag_release\n if: github.event_name == 'release' && startsWith(github.event.release.tag_name, '0.')\n uses: docker/build-push-action@v7\n with:\n context: ./\n file: ./Dockerfile\n push: true\n tags: ${{ steps.meta.outputs.tags }}\n labels: ${{ steps.meta.outputs.labels }}\n platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8\n cache-from: type=gha\n cache-to: type=gha,mode=max\n# Looks like this was disabled\n# provenance: false\n\n - name: Image digest\n run: echo step SHA ${{ steps.vars.outputs.sha_short }} tag ${{steps.vars.outputs.tag}} branch ${{steps.vars.outputs.branch}} digest ${{ steps.docker_build.outputs.digest }}\n\n" }, { "path": ".github/workflows/pypi-release.yml", "content": "name: Publish Python 🐍distribution 📦 to PyPI and TestPyPI\n\non: push\njobs:\n build:\n name: Build distribution 📦\n runs-on: ubuntu-latest\n\n steps:\n - uses: actions/checkout@v6\n - name: Set up Python\n uses: actions/setup-python@v6\n with:\n python-version: \"3.11\"\n - name: Install pypa/build\n run: >-\n python3 -m\n pip install\n build\n --user\n - name: Build a binary wheel and a source tarball\n run: python3 -m build\n - name: Store the distribution packages\n uses: actions/upload-artifact@v7\n with:\n name: python-package-distributions\n path: dist/\n\n\n test-pypi-package:\n name: Test the built package works basically.\n runs-on: ubuntu-latest\n needs:\n - build\n steps:\n - name: Download all the dists\n uses: actions/download-artifact@v8\n with:\n name: python-package-distributions\n path: dist/\n - name: Set up Python 3.11\n uses: actions/setup-python@v6\n with:\n python-version: '3.11'\n\n - name: Test that the basic pip built package runs without error\n run: |\n set -ex\n ls -alR \n \n # Install the first wheel found in dist/\n WHEEL=$(find dist -type f -name \"*.whl\" -print -quit)\n echo Installing $WHEEL\n python3 -m pip install --upgrade pip\n python3 -m pip install \"$WHEEL\"\n changedetection.io -d /tmp -p 10000 &\n \n sleep 3\n curl --retry-connrefused --retry 6 http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null\n curl --retry-connrefused --retry 6 http://127.0.0.1:10000/ >/dev/null\n \n # --- API test ---\n # This also means that the docs/api-spec.yml was shipped and could be read\n test -f /tmp/changedetection.json\n API_KEY=$(jq -r '.. | .api_access_token? // empty' /tmp/changedetection.json)\n echo Test API KEY is $API_KEY\n curl -X POST \"http://127.0.0.1:10000/api/v1/watch\" \\\n -H \"x-api-key: ${API_KEY}\" \\\n -H \"Content-Type: application/json\" \\\n --show-error --fail \\\n --retry 6 --retry-delay 1 --retry-connrefused \\\n -d '{\n \"url\": \"https://example.com\",\n \"title\": \"Example Site Monitor\",\n \"time_between_check\": { \"hours\": 1 }\n }'\n \n killall changedetection.io\n\n\n publish-to-pypi:\n name: >-\n Publish Python 🐍 distribution 📦 to PyPI\n if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes\n needs:\n - test-pypi-package\n runs-on: ubuntu-latest\n environment:\n name: release\n url: https://pypi.org/p/changedetection.io\n permissions:\n id-token: write # IMPORTANT: mandatory for trusted publishing\n\n steps:\n - name: Download all the dists\n uses: actions/download-artifact@v8\n with:\n name: python-package-distributions\n path: dist/\n - name: Publish distribution 📦 to PyPI\n uses: pypa/gh-action-pypi-publish@release/v1\n" }, { "path": ".github/workflows/test-container-build.yml", "content": "name: ChangeDetection.io Container Build Test\n\n# Triggers the workflow on push or pull request events\n\n# This line doesnt work, even tho it is the documented one\n#on: [push, pull_request]\n\non:\n push:\n paths:\n - requirements.txt\n - Dockerfile\n - .github/workflows/*\n - .github/test/Dockerfile*\n\n pull_request:\n paths:\n - requirements.txt\n - Dockerfile\n - .github/workflows/*\n - .github/test/Dockerfile*\n\n # Changes to requirements.txt packages and Dockerfile may or may not always be compatible with arm etc, so worth testing\n # @todo: some kind of path filter for requirements.txt and Dockerfile\njobs:\n builder:\n name: Build ${{ matrix.platform }} (${{ matrix.dockerfile == './Dockerfile' && 'main' || 'alpine' }})\n runs-on: ubuntu-latest\n strategy:\n matrix:\n include:\n # Main Dockerfile platforms\n - platform: linux/amd64\n dockerfile: ./Dockerfile\n - platform: linux/arm64\n dockerfile: ./Dockerfile\n - platform: linux/arm/v7\n dockerfile: ./Dockerfile\n - platform: linux/arm/v8\n dockerfile: ./Dockerfile\n # Alpine Dockerfile platforms (musl via alpine check)\n - platform: linux/amd64\n dockerfile: ./.github/test/Dockerfile-alpine\n - platform: linux/arm64\n dockerfile: ./.github/test/Dockerfile-alpine\n steps:\n - uses: actions/checkout@v6\n - name: Set up Python 3.11\n uses: actions/setup-python@v6\n with:\n python-version: 3.11\n\n - name: Cache pip packages\n uses: actions/cache@v5\n with:\n path: ~/.cache/pip\n key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}\n restore-keys: |\n ${{ runner.os }}-pip-\n\n # Just test that the build works, some libraries won't compile on ARM/rPi etc\n - name: Set up QEMU\n uses: docker/setup-qemu-action@v4\n with:\n image: tonistiigi/binfmt:latest\n platforms: all\n\n - name: Set up Docker Buildx\n id: buildx\n uses: docker/setup-buildx-action@v4\n with:\n install: true\n version: latest\n driver-opts: image=moby/buildkit:master\n\n - name: Test that the docker containers can build (${{ matrix.platform }} - ${{ matrix.dockerfile }})\n id: docker_build\n uses: docker/build-push-action@v7\n # https://github.com/docker/build-push-action#customizing\n with:\n context: ./\n file: ${{ matrix.dockerfile }}\n platforms: ${{ matrix.platform }}\n cache-from: type=gha\n cache-to: type=gha,mode=max\n\n" }, { "path": ".github/workflows/test-only.yml", "content": "name: ChangeDetection.io App Test\n\n# Triggers the workflow on push or pull request events\non: [push, pull_request]\n\njobs:\n lint-code:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v6\n - name: Lint with Ruff\n run: |\n pip install ruff\n # Check for syntax errors and undefined names\n ruff check . --select E9,F63,F7,F82\n # Complete check with errors treated as warnings\n ruff check . --exit-zero\n - name: Validate OpenAPI spec\n run: |\n pip install openapi-spec-validator\n python3 -c \"from openapi_spec_validator import validate_spec; import yaml; validate_spec(yaml.safe_load(open('docs/api-spec.yaml')))\"\n\n test-application-3-10:\n # Only run on push to master (including PR merges)\n if: github.event_name == 'push' && github.ref == 'refs/heads/master'\n needs: lint-code\n uses: ./.github/workflows/test-stack-reusable-workflow.yml\n with:\n python-version: '3.10'\n\n\n test-application-3-11:\n # Always run\n needs: lint-code\n uses: ./.github/workflows/test-stack-reusable-workflow.yml\n with:\n python-version: '3.11'\n\n test-application-3-12:\n # Only run on push to master (including PR merges)\n if: github.event_name == 'push' && github.ref == 'refs/heads/master'\n needs: lint-code\n uses: ./.github/workflows/test-stack-reusable-workflow.yml\n with:\n python-version: '3.12'\n skip-pypuppeteer: true\n\n test-application-3-13:\n # Only run on push to master (including PR merges)\n if: github.event_name == 'push' && github.ref == 'refs/heads/master'\n needs: lint-code\n uses: ./.github/workflows/test-stack-reusable-workflow.yml\n with:\n python-version: '3.13'\n skip-pypuppeteer: true\n\n\n test-application-3-14:\n #if: github.event_name == 'push' && github.ref == 'refs/heads/master'\n needs: lint-code\n uses: ./.github/workflows/test-stack-reusable-workflow.yml\n with:\n python-version: '3.14'\n skip-pypuppeteer: false\n" }, { "path": ".github/workflows/test-stack-reusable-workflow.yml", "content": "name: ChangeDetection.io App Test\n\non:\n workflow_call:\n inputs:\n python-version:\n description: 'Python version to use'\n required: true\n type: string\n default: '3.11'\n skip-pypuppeteer:\n description: 'Skip PyPuppeteer (not supported in 3.11/3.12)'\n required: false\n type: boolean\n default: false\n\njobs:\n # Build the Docker image once and share it with all test jobs\n build:\n runs-on: ubuntu-latest\n env:\n PYTHON_VERSION: ${{ inputs.python-version }}\n steps:\n - uses: actions/checkout@v6\n\n - name: Set up Python ${{ env.PYTHON_VERSION }}\n uses: actions/setup-python@v6\n with:\n python-version: ${{ env.PYTHON_VERSION }}\n\n - name: Cache pip packages\n uses: actions/cache@v5\n with:\n path: ~/.cache/pip\n key: ${{ runner.os }}-pip-py${{ env.PYTHON_VERSION }}-${{ hashFiles('requirements.txt') }}\n restore-keys: |\n ${{ runner.os }}-pip-py${{ env.PYTHON_VERSION }}-\n ${{ runner.os }}-pip-\n\n - name: Get current date for cache key\n id: date\n run: echo \"date=$(date +'%Y-%m-%d')\" >> $GITHUB_OUTPUT\n\n - name: Set up Docker Buildx\n uses: docker/setup-buildx-action@v4\n\n - name: Build changedetection.io container for testing under Python ${{ env.PYTHON_VERSION }}\n uses: docker/build-push-action@v7\n with:\n context: ./\n file: ./Dockerfile\n build-args: |\n PYTHON_VERSION=${{ env.PYTHON_VERSION }}\n LOGGER_LEVEL=TRACE\n tags: test-changedetectionio\n load: true\n cache-from: type=gha,scope=build-${{ github.ref_name }}-py${{ env.PYTHON_VERSION }}-${{ hashFiles('requirements.txt', 'Dockerfile') }}-${{ steps.date.outputs.date }}\n cache-to: type=gha,mode=max,scope=build-${{ github.ref_name }}-py${{ env.PYTHON_VERSION }}-${{ hashFiles('requirements.txt', 'Dockerfile') }}-${{ steps.date.outputs.date }}\n\n - name: Verify build\n run: |\n echo \"---- Built for Python ${{ env.PYTHON_VERSION }} -----\"\n docker run test-changedetectionio bash -c 'pip list'\n\n - name: We should be Python ${{ env.PYTHON_VERSION }} ...\n run: |\n docker run test-changedetectionio bash -c 'python3 --version'\n\n - name: Save Docker image\n run: |\n docker save test-changedetectionio -o /tmp/test-changedetectionio.tar\n\n - name: Upload Docker image artifact\n uses: actions/upload-artifact@v7\n with:\n name: test-changedetectionio-${{ env.PYTHON_VERSION }}\n path: /tmp/test-changedetectionio.tar\n retention-days: 1\n\n # Unit tests (lightweight, no ancillary services needed)\n unit-tests:\n runs-on: ubuntu-latest\n needs: build\n timeout-minutes: 10\n env:\n PYTHON_VERSION: ${{ inputs.python-version }}\n steps:\n - uses: actions/checkout@v6\n\n - name: Download Docker image artifact\n uses: actions/download-artifact@v8\n with:\n name: test-changedetectionio-${{ env.PYTHON_VERSION }}\n path: /tmp\n\n - name: Load Docker image\n run: |\n docker load -i /tmp/test-changedetectionio.tar\n\n - name: Run Unit Tests\n run: |\n docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff'\n docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model'\n docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_jinja2_security'\n docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_semver'\n docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_html_to_text' \n\n # Basic pytest tests with ancillary services\n basic-tests:\n runs-on: ubuntu-latest\n needs: build\n timeout-minutes: 25\n env:\n PYTHON_VERSION: ${{ inputs.python-version }}\n steps:\n - uses: actions/checkout@v6\n\n - name: Download Docker image artifact\n uses: actions/download-artifact@v8\n with:\n name: test-changedetectionio-${{ env.PYTHON_VERSION }}\n path: /tmp\n\n - name: Load Docker image\n run: |\n docker load -i /tmp/test-changedetectionio.tar\n\n - name: Test built container with Pytest\n run: |\n docker network inspect changedet-network >/dev/null 2>&1 || docker network create changedet-network\n docker run --name test-cdio-basic-tests --network changedet-network test-changedetectionio bash -c 'cd changedetectionio && ./run_basic_tests.sh'\n\n - name: Test CLI options\n run: |\n docker network inspect changedet-network >/dev/null 2>&1 || docker network create changedet-network\n docker run --name test-cdio-cli-opts --network changedet-network test-changedetectionio bash -c 'changedetectionio/test_cli_opts.sh' &> cli-opts-output.txt\n echo \"=== CLI Options Test Output ===\"\n cat cli-opts-output.txt\n\n - name: CLI Memory Test\n run: |\n echo \"=== Checking CLI batch mode memory usage ===\"\n # Extract RSS memory value from output\n RSS_MB=$(grep -oP \"Memory consumption before worker shutdown: RSS=\\K[\\d.]+\" cli-opts-output.txt | head -1 || echo \"0\")\n echo \"RSS Memory: ${RSS_MB} MB\"\n\n # Check if RSS is less than 100MB\n if [ -n \"$RSS_MB\" ]; then\n if (( $(echo \"$RSS_MB < 100\" | bc -l) )); then\n echo \"✓ Memory usage is acceptable: ${RSS_MB} MB < 100 MB\"\n else\n echo \"✗ Memory usage too high: ${RSS_MB} MB >= 100 MB\"\n exit 1\n fi\n else\n echo \"⚠ Could not extract memory usage, skipping check\"\n fi\n\n - name: Extract memory report and logs\n if: always()\n uses: ./.github/actions/extract-memory-report\n with:\n container-name: test-cdio-basic-tests\n python-version: ${{ env.PYTHON_VERSION }}\n\n - name: Store test artifacts\n if: always()\n uses: actions/upload-artifact@v7\n with:\n name: test-cdio-basic-tests-output-py${{ env.PYTHON_VERSION }}\n path: output-logs\n\n - name: Store CLI test output\n if: always()\n uses: actions/upload-artifact@v7\n with:\n name: test-cdio-cli-opts-output-py${{ env.PYTHON_VERSION }}\n path: cli-opts-output.txt\n\n # Playwright tests\n playwright-tests:\n runs-on: ubuntu-latest\n needs: build\n timeout-minutes: 10\n env:\n PYTHON_VERSION: ${{ inputs.python-version }}\n steps:\n - uses: actions/checkout@v6\n\n - name: Download Docker image artifact\n uses: actions/download-artifact@v8\n with:\n name: test-changedetectionio-${{ env.PYTHON_VERSION }}\n path: /tmp\n\n - name: Load Docker image\n run: |\n docker load -i /tmp/test-changedetectionio.tar\n\n - name: Spin up ancillary services\n run: |\n docker network create changedet-network\n docker run --network changedet-network -d -e \"LOG_LEVEL=TRACE\" --cap-add=SYS_ADMIN --name sockpuppetbrowser --hostname sockpuppetbrowser --rm -p 3000:3000 dgtlmoon/sockpuppetbrowser:latest\n docker run --network changedet-network -d -e \"LOG_LEVEL=TRACE\" --cap-add=SYS_ADMIN --name sockpuppetbrowser-custom-url --hostname sockpuppetbrowser-custom-url -p 3001:3000 --rm dgtlmoon/sockpuppetbrowser:latest\n\n - name: Playwright - Specific tests in built container\n run: |\n docker run --rm -e \"FLASK_SERVER_NAME=cdio\" -e \"PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000\" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest -vv --capture=tee-sys --showlocals --tb=long --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'\n docker run --rm -e \"FLASK_SERVER_NAME=cdio\" -e \"PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000\" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest -vv --capture=tee-sys --showlocals --tb=long --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'\n docker run --rm -e \"FLASK_SERVER_NAME=cdio\" -e \"PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000\" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest -vv --capture=tee-sys --showlocals --tb=long --live-server-host=0.0.0.0 --live-server-port=5004 tests/visualselector/test_fetch_data.py'\n docker run --rm -e \"FLASK_SERVER_NAME=cdio\" -e \"PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000\" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest -vv --capture=tee-sys --showlocals --tb=long --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_custom_js_before_content.py'\n\n - name: Playwright - Headers and requests\n run: |\n docker run --name \"changedet\" --hostname changedet --rm -e \"FLASK_SERVER_NAME=changedet\" -e \"PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true\" --network changedet-network test-changedetectionio bash -c 'find .; cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py; pwd;find .'\n\n - name: Playwright - Restock detection\n run: |\n docker run --rm --name \"changedet\" -e \"FLASK_SERVER_NAME=changedet\" -e \"PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000\" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'\n\n # Pyppeteer tests\n pyppeteer-tests:\n runs-on: ubuntu-latest\n needs: build\n if: ${{ inputs.skip-pypuppeteer == false }}\n timeout-minutes: 10\n env:\n PYTHON_VERSION: ${{ inputs.python-version }}\n steps:\n - uses: actions/checkout@v6\n\n - name: Download Docker image artifact\n uses: actions/download-artifact@v8\n with:\n name: test-changedetectionio-${{ env.PYTHON_VERSION }}\n path: /tmp\n\n - name: Load Docker image\n run: |\n docker load -i /tmp/test-changedetectionio.tar\n\n - name: Spin up ancillary services\n run: |\n docker network create changedet-network\n docker run --network changedet-network -d -e \"LOG_LEVEL=TRACE\" --cap-add=SYS_ADMIN --name sockpuppetbrowser --hostname sockpuppetbrowser --rm -p 3000:3000 dgtlmoon/sockpuppetbrowser:latest\n\n - name: Pyppeteer - Specific tests in built container\n run: |\n docker run --rm -e \"FLASK_SERVER_NAME=cdio\" -e \"FAST_PUPPETEER_CHROME_FETCHER=True\" -e \"PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000\" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_content.py'\n docker run --rm -e \"FLASK_SERVER_NAME=cdio\" -e \"FAST_PUPPETEER_CHROME_FETCHER=True\" -e \"PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000\" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_errorhandling.py'\n docker run --rm -e \"FLASK_SERVER_NAME=cdio\" -e \"FAST_PUPPETEER_CHROME_FETCHER=True\" -e \"PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000\" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/visualselector/test_fetch_data.py'\n docker run --rm -e \"FLASK_SERVER_NAME=cdio\" -e \"FAST_PUPPETEER_CHROME_FETCHER=True\" -e \"PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000\" --network changedet-network --hostname=cdio test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/fetchers/test_custom_js_before_content.py'\n\n - name: Pyppeteer - Headers and requests checks\n run: |\n docker run --name \"changedet\" --hostname changedet --rm -e \"FAST_PUPPETEER_CHROME_FETCHER=True\" -e \"FLASK_SERVER_NAME=changedet\" -e \"PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000?dumpio=true\" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py'\n\n - name: Pyppeteer - Restock detection\n run: |\n docker run --rm --name \"changedet\" -e \"FLASK_SERVER_NAME=changedet\" -e \"FAST_PUPPETEER_CHROME_FETCHER=True\" -e \"PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000\" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest --live-server-port=5004 --live-server-host=0.0.0.0 tests/restock/test_restock.py'\n\n # Selenium tests\n selenium-tests:\n runs-on: ubuntu-latest\n needs: build\n timeout-minutes: 10\n env:\n PYTHON_VERSION: ${{ inputs.python-version }}\n steps:\n - uses: actions/checkout@v6\n\n - name: Download Docker image artifact\n uses: actions/download-artifact@v8\n with:\n name: test-changedetectionio-${{ env.PYTHON_VERSION }}\n path: /tmp\n\n - name: Load Docker image\n run: |\n docker load -i /tmp/test-changedetectionio.tar\n\n - name: Spin up ancillary services\n run: |\n docker network create changedet-network\n docker run --network changedet-network -d --hostname selenium -p 4444:4444 --rm --shm-size=\"2g\" selenium/standalone-chrome:4\n sleep 3\n\n - name: Specific tests for headers and requests checks with Selenium\n run: |\n\n docker run --name \"changedet\" --hostname changedet --rm -e \"FLASK_SERVER_NAME=changedet\" -e \"WEBDRIVER_URL=http://selenium:4444/wd/hub\" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio; pytest --live-server-host=0.0.0.0 --live-server-port=5004 tests/test_request.py'\n\n - name: Specific tests in built container for Selenium\n run: |\n docker run --rm -e \"WEBDRIVER_URL=http://selenium:4444/wd/hub\" --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/fetchers/test_content.py && pytest tests/test_errorhandling.py'\n\n\n # SMTP tests\n smtp-tests:\n runs-on: ubuntu-latest\n needs: build\n timeout-minutes: 10\n env:\n PYTHON_VERSION: ${{ inputs.python-version }}\n steps:\n - uses: actions/checkout@v6\n\n - name: Download Docker image artifact\n uses: actions/download-artifact@v8\n with:\n name: test-changedetectionio-${{ env.PYTHON_VERSION }}\n path: /tmp\n\n - name: Load Docker image\n run: |\n docker load -i /tmp/test-changedetectionio.tar\n\n - name: Spin up SMTP test server\n run: |\n docker network create changedet-network\n docker run --network changedet-network -d -p 11025:11025 -p 11080:11080 --hostname mailserver test-changedetectionio bash -c 'pip3 install aiosmtpd && python changedetectionio/tests/smtp/smtp-test-server.py'\n\n - name: Test SMTP notification mime types\n run: |\n docker run --rm --network changedet-network test-changedetectionio bash -c 'cd changedetectionio;pytest tests/smtp/test_notification_smtp.py'\n\n nginx-reverse-proxy:\n runs-on: ubuntu-latest\n needs: build\n timeout-minutes: 10\n env:\n PYTHON_VERSION: ${{ inputs.python-version }}\n steps:\n - uses: actions/checkout@v6\n\n - name: Download Docker image artifact\n uses: actions/download-artifact@v8\n with:\n name: test-changedetectionio-${{ env.PYTHON_VERSION }}\n path: /tmp\n\n - name: Load Docker image\n run: |\n docker load -i /tmp/test-changedetectionio.tar\n\n - name: Spin up services\n run: |\n docker network create changedet-network\n\n # Start changedetection.io container with X-Forwarded headers support\n docker run --name changedet-app --hostname changedet-app --network changedet-network \\\n -e USE_X_SETTINGS=true \\\n -d test-changedetectionio\n sleep 3\n\n - name: Start nginx reverse proxy\n run: |\n # Start nginx with our test configuration\n docker run --name nginx-proxy --network changedet-network -d -p 8080:80 --rm \\\n -v ${{ github.workspace }}/.github/nginx-reverse-proxy-test.conf:/etc/nginx/conf.d/default.conf:ro \\\n nginx:alpine\n sleep 2\n\n - name: Test reverse proxy - root path\n run: |\n echo \"=== Testing nginx reverse proxy at root path ===\"\n curl --retry-connrefused --retry 6 -s http://localhost:8080/ > /tmp/nginx-test-root.html\n\n # Check for changedetection.io UI elements\n if grep -q \"checkbox-uuid\" /tmp/nginx-test-root.html; then\n echo \"✓ Found checkbox-uuid in response\"\n else\n echo \"ERROR: checkbox-uuid not found in response\"\n cat /tmp/nginx-test-root.html\n exit 1\n fi\n\n # Check for watchlist content\n if grep -q -i \"watch\" /tmp/nginx-test-root.html; then\n echo \"✓ Found watch/watchlist content in response\"\n else\n echo \"ERROR: watchlist content not found\"\n cat /tmp/nginx-test-root.html\n exit 1\n fi\n\n echo \"✓ Root path reverse proxy working correctly\"\n\n - name: Test reverse proxy - subpath with X-Forwarded-Prefix\n run: |\n echo \"=== Testing nginx reverse proxy at subpath /changedet-sub/ ===\"\n curl --retry-connrefused --retry 6 -s http://localhost:8080/changedet-sub/ > /tmp/nginx-test-subpath.html\n\n # Check for changedetection.io UI elements\n if grep -q \"checkbox-uuid\" /tmp/nginx-test-subpath.html; then\n echo \"✓ Found checkbox-uuid in subpath response\"\n else\n echo \"ERROR: checkbox-uuid not found in subpath response\"\n cat /tmp/nginx-test-subpath.html\n exit 1\n fi\n\n echo \"✓ Subpath reverse proxy working correctly\"\n\n - name: Test API through reverse proxy subpath\n run: |\n echo \"=== Testing API endpoints through nginx subpath /changedet-sub/ ===\"\n\n # Extract API key from the changedetection.io datastore\n API_KEY=$(docker exec changedet-app cat /datastore/changedetection.json | grep -o '\"api_access_token\": *\"[^\"]*\"' | cut -d'\"' -f4)\n\n if [ -z \"$API_KEY\" ]; then\n echo \"ERROR: Could not extract API key from datastore\"\n docker exec changedet-app cat /datastore/changedetection.json\n exit 1\n fi\n\n echo \"✓ Extracted API key: ${API_KEY:0:8}...\"\n\n # Create a watch via API through nginx proxy subpath\n echo \"Creating watch via POST to /changedet-sub/api/v1/watch\"\n RESPONSE=$(curl -s -w \"\\n%{http_code}\" -X POST \"http://localhost:8080/changedet-sub/api/v1/watch\" \\\n -H \"x-api-key: ${API_KEY}\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"url\": \"https://example.com/test-nginx-proxy\",\n \"tag\": \"nginx-test\"\n }')\n\n HTTP_CODE=$(echo \"$RESPONSE\" | tail -n1)\n BODY=$(echo \"$RESPONSE\" | head -n-1)\n\n if [ \"$HTTP_CODE\" != \"201\" ]; then\n echo \"ERROR: Expected HTTP 201, got $HTTP_CODE\"\n echo \"Response: $BODY\"\n exit 1\n fi\n\n echo \"✓ Watch created successfully (HTTP 201)\"\n\n # Extract the watch UUID from response\n WATCH_UUID=$(echo \"$BODY\" | grep -o '\"uuid\": *\"[^\"]*\"' | cut -d'\"' -f4)\n echo \"✓ Watch UUID: $WATCH_UUID\"\n\n # Update the watch via PUT through nginx proxy subpath\n echo \"Updating watch via PUT to /changedet-sub/api/v1/watch/${WATCH_UUID}\"\n RESPONSE=$(curl -s -w \"\\n%{http_code}\" -X PUT \"http://localhost:8080/changedet-sub/api/v1/watch/${WATCH_UUID}\" \\\n -H \"x-api-key: ${API_KEY}\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"paused\": true\n }')\n\n HTTP_CODE=$(echo \"$RESPONSE\" | tail -n1)\n BODY=$(echo \"$RESPONSE\" | head -n-1)\n\n if [ \"$HTTP_CODE\" != \"200\" ]; then\n echo \"ERROR: Expected HTTP 200, got $HTTP_CODE\"\n echo \"Response: $BODY\"\n exit 1\n fi\n\n if echo \"$BODY\" | grep -q 'OK'; then\n echo \"✓ Watch updated successfully (HTTP 200, response: OK)\"\n else\n echo \"ERROR: Expected response 'OK', got: $BODY\"\n echo \"Response: $BODY\"\n exit 1\n fi\n\n # Verify the watch is paused via GET\n echo \"Verifying watch is paused via GET\"\n RESPONSE=$(curl -s \"http://localhost:8080/changedet-sub/api/v1/watch/${WATCH_UUID}\" \\\n -H \"x-api-key: ${API_KEY}\")\n\n if echo \"$RESPONSE\" | grep -q '\"paused\": *true'; then\n echo \"✓ Watch is paused as expected\"\n else\n echo \"ERROR: Watch paused state not confirmed\"\n echo \"Response: $RESPONSE\"\n exit 1\n fi\n\n echo \"✓ API tests through nginx subpath completed successfully\"\n\n - name: Cleanup nginx test\n if: always()\n run: |\n docker logs nginx-proxy || true\n docker logs changedet-app || true\n docker stop nginx-proxy changedet-app || true\n docker rm nginx-proxy changedet-app || true\n\n\n\n # Proxy tests\n proxy-tests:\n runs-on: ubuntu-latest\n needs: build\n timeout-minutes: 10\n env:\n PYTHON_VERSION: ${{ inputs.python-version }}\n steps:\n - uses: actions/checkout@v6\n\n - name: Download Docker image artifact\n uses: actions/download-artifact@v8\n with:\n name: test-changedetectionio-${{ env.PYTHON_VERSION }}\n path: /tmp\n\n - name: Load Docker image\n run: |\n docker load -i /tmp/test-changedetectionio.tar\n\n - name: Spin up services\n run: |\n docker network create changedet-network\n docker run --network changedet-network -d --hostname selenium -p 4444:4444 --rm --shm-size=\"2g\" selenium/standalone-chrome:4\n docker run --network changedet-network -d -e \"LOG_LEVEL=TRACE\" --cap-add=SYS_ADMIN --name sockpuppetbrowser --hostname sockpuppetbrowser --rm -p 3000:3000 dgtlmoon/sockpuppetbrowser:latest\n docker run --network changedet-network -d -e \"LOG_LEVEL=TRACE\" --cap-add=SYS_ADMIN --name sockpuppetbrowser-custom-url --hostname sockpuppetbrowser-custom-url -p 3001:3000 --rm dgtlmoon/sockpuppetbrowser:latest\n\n - name: Test proxy Squid style interaction\n run: |\n cd changedetectionio\n ./run_proxy_tests.sh\n docker ps\n cd ..\n\n - name: Test proxy SOCKS5 style interaction\n run: |\n cd changedetectionio\n ./run_socks_proxy_tests.sh\n cd ..\n\n # Custom browser URL tests\n custom-browser-tests:\n runs-on: ubuntu-latest\n needs: build\n timeout-minutes: 10\n env:\n PYTHON_VERSION: ${{ inputs.python-version }}\n steps:\n - uses: actions/checkout@v6\n\n - name: Download Docker image artifact\n uses: actions/download-artifact@v8\n with:\n name: test-changedetectionio-${{ env.PYTHON_VERSION }}\n path: /tmp\n\n - name: Load Docker image\n run: |\n docker load -i /tmp/test-changedetectionio.tar\n\n - name: Spin up ancillary services\n run: |\n docker network create changedet-network\n docker run --network changedet-network -d -e \"LOG_LEVEL=TRACE\" --cap-add=SYS_ADMIN --name sockpuppetbrowser --hostname sockpuppetbrowser --rm -p 3000:3000 dgtlmoon/sockpuppetbrowser:latest\n docker run --network changedet-network -d -e \"LOG_LEVEL=TRACE\" --cap-add=SYS_ADMIN --name sockpuppetbrowser-custom-url --hostname sockpuppetbrowser-custom-url -p 3001:3000 --rm dgtlmoon/sockpuppetbrowser:latest\n\n - name: Test custom browser URL\n run: |\n cd changedetectionio\n ./run_custom_browser_url_tests.sh\n\n processor-plugin-tests:\n runs-on: ubuntu-latest\n needs: build\n timeout-minutes: 20\n env:\n PYTHON_VERSION: ${{ inputs.python-version }}\n steps:\n - uses: actions/checkout@v6\n\n - name: Download Docker image artifact\n uses: actions/download-artifact@v8\n with:\n name: test-changedetectionio-${{ env.PYTHON_VERSION }}\n path: /tmp\n\n - name: Load Docker image\n run: |\n docker load -i /tmp/test-changedetectionio.tar\n\n - name: Basic processor plugin registration and checks\n run: |\n docker run -e EXTRA_PACKAGES=changedetection.io-osint-processor test-changedetectionio bash -c 'cd changedetectionio;pytest -vvv -s tests/plugins/test_processor.py::test_check_plugin_processor'\n\n # Container startup tests\n container-tests:\n runs-on: ubuntu-latest\n needs: build\n timeout-minutes: 10\n env:\n PYTHON_VERSION: ${{ inputs.python-version }}\n steps:\n - uses: actions/checkout@v6\n\n - name: Download Docker image artifact\n uses: actions/download-artifact@v8\n with:\n name: test-changedetectionio-${{ env.PYTHON_VERSION }}\n path: /tmp\n\n - name: Load Docker image\n run: |\n docker load -i /tmp/test-changedetectionio.tar\n\n - name: Test container starts+runs basically without error\n run: |\n docker run --name test-changedetectionio -p 5556:5000 -d test-changedetectionio\n sleep 3\n curl --retry-connrefused --retry 6 -s http://localhost:5556 |grep -q checkbox-uuid\n curl --retry-connrefused --retry 6 -s -g -6 \"http://[::1]:5556\"|grep -q checkbox-uuid\n docker logs test-changedetectionio 2>/dev/null | grep 'TRACE log is enabled' || exit 1\n docker logs test-changedetectionio 2>/dev/null | grep 'DEBUG' || exit 1\n docker kill test-changedetectionio\n\n - name: Test HTTPS SSL mode\n run: |\n openssl req -x509 -newkey rsa:4096 -keyout privkey.pem -out cert.pem -days 365 -nodes -subj \"/CN=localhost\"\n docker run --name test-changedetectionio-ssl --rm -e SSL_CERT_FILE=cert.pem -e SSL_PRIVKEY_FILE=privkey.pem -p 5000:5000 -v ./cert.pem:/app/cert.pem -v ./privkey.pem:/app/privkey.pem -d test-changedetectionio\n sleep 3\n curl --retry-connrefused --retry 6 -k https://localhost:5000 -v|grep -q checkbox-uuid\n docker kill test-changedetectionio-ssl\n\n - name: Test IPv6 Mode\n run: |\n docker run --name test-changedetectionio-ipv6 --rm -p 5000:5000 -e LISTEN_HOST=:: -d test-changedetectionio\n sleep 3\n curl --retry-connrefused --retry 6 http://[::1]:5000 -v|grep -q checkbox-uuid\n docker kill test-changedetectionio-ipv6\n\n # Signal tests\n signal-tests:\n runs-on: ubuntu-latest\n needs: build\n timeout-minutes: 10\n env:\n PYTHON_VERSION: ${{ inputs.python-version }}\n steps:\n - uses: actions/checkout@v6\n\n - name: Download Docker image artifact\n uses: actions/download-artifact@v8\n with:\n name: test-changedetectionio-${{ env.PYTHON_VERSION }}\n path: /tmp\n\n - name: Load Docker image\n run: |\n docker load -i /tmp/test-changedetectionio.tar\n\n - name: Test SIGTERM and SIGINT signal shutdown\n run: |\n echo SIGINT Shutdown request test\n docker run --name sig-test -d test-changedetectionio\n sleep 3\n echo \">>> Sending SIGINT to sig-test container\"\n docker kill --signal=SIGINT sig-test\n sleep 3\n docker ps\n docker logs sig-test 2>&1 | grep 'Shutdown: Got Signal - SIGINT' || exit 1\n test -z \"`docker ps|grep sig-test`\"\n if [ $? -ne 0 ]; then\n echo \"Looks like container was running when it shouldnt be\"\n docker ps\n exit 1\n fi\n docker rm sig-test\n\n echo SIGTERM Shutdown request test\n docker run --name sig-test -d test-changedetectionio\n sleep 3\n echo \">>> Sending SIGTERM to sig-test container\"\n docker kill --signal=SIGTERM sig-test\n sleep 3\n docker ps\n docker logs sig-test 2>&1 | grep 'Shutdown: Got Signal - SIGTERM' || exit 1\n test -z \"`docker ps|grep sig-test`\"\n if [ $? -ne 0 ]; then\n echo \"Looks like container was running when it shouldnt be\"\n docker ps\n exit 1\n fi\n docker rm sig-test\n\n # Upgrade path test\n upgrade-path-test:\n runs-on: ubuntu-latest\n needs: build\n timeout-minutes: 25\n env:\n PYTHON_VERSION: ${{ inputs.python-version }}\n steps:\n - uses: actions/checkout@v6\n with:\n fetch-depth: 0 # Fetch all history and tags for upgrade testing\n\n - name: Set up Python ${{ env.PYTHON_VERSION }}\n uses: actions/setup-python@v6\n with:\n python-version: ${{ env.PYTHON_VERSION }}\n\n - name: Check upgrade works without error\n run: |\n echo \"=== Testing upgrade path from 0.49.1 to ${{ github.ref_name }} (${{ github.sha }}) ===\"\n sudo apt-get update && sudo apt-get install -y --no-install-recommends \\\n g++ \\\n gcc \\\n libc-dev \\\n libffi-dev \\\n libjpeg-dev \\\n libssl-dev \\\n libxslt-dev \\\n make \\\n patch \\\n pkg-config \\\n zlib1g-dev\n \n # Checkout old version and create datastore\n git checkout 0.49.1\n python3 -m venv .venv\n source .venv/bin/activate\n pip install -r requirements.txt\n pip install 'pyOpenSSL>=23.2.0'\n\n echo \"=== Running version 0.49.1 to create datastore ===\"\n ALLOW_IANA_RESTRICTED_ADDRESSES=true python3 ./changedetection.py -C -d /tmp/data &\n APP_PID=$!\n\n # Wait for app to be ready\n echo \"Waiting for 0.49.1 to be ready...\"\n sleep 6\n\n # Extract API key from datastore (0.49.1 uses url-watches.json)\n API_KEY=$(jq -r '.settings.application.api_access_token // empty' /tmp/data/url-watches.json)\n echo \"API Key: ${API_KEY:0:8}...\"\n\n # Create a watch with tag \"github-group-test\" via API\n echo \"Creating test watch with tag via API...\"\n curl -X POST \"http://127.0.0.1:5000/api/v1/watch\" \\\n -H \"x-api-key: ${API_KEY}\" \\\n -H \"Content-Type: application/json\" \\\n --show-error --fail \\\n --retry 6 --retry-delay 1 --retry-connrefused \\\n -d '{\n \"url\": \"https://example.com/upgrade-test\",\n \"tag\": \"github-group-test\"\n }'\n\n echo \"✓ Created watch with tag 'github-group-test'\"\n\n # Create a specific test URL watch\n echo \"Creating test URL watch via API...\"\n curl -X POST \"http://127.0.0.1:5000/api/v1/watch\" \\\n -H \"x-api-key: ${API_KEY}\" \\\n -H \"Content-Type: application/json\" \\\n --show-error --fail \\\n -d '{\n \"url\": \"http://localhost/test.txt\"\n }'\n\n echo \"✓ Created watch for 'http://localhost/test.txt' in version 0.49.1\"\n\n # Stop the old version gracefully\n kill $APP_PID\n wait $APP_PID || true\n echo \"✓ Version 0.49.1 stopped\"\n\n # Upgrade to current version (use commit SHA since we're in detached HEAD)\n echo \"Upgrading to commit ${{ github.sha }}\"\n git checkout ${{ github.sha }}\n pip install -r requirements.txt\n\n echo \"=== Running current version (commit ${{ github.sha }}) with old datastore (testing mode) ===\"\n ALLOW_IANA_RESTRICTED_ADDRESSES=true TESTING_SHUTDOWN_AFTER_DATASTORE_LOAD=1 python3 ./changedetection.py -d /tmp/data > /tmp/upgrade-test.log 2>&1\n\n echo \"=== Upgrade test output ===\"\n cat /tmp/upgrade-test.log\n echo \"✓ Datastore upgraded successfully\"\n\n # Now start the current version normally to verify the tag survived\n echo \"=== Starting current version to verify tag exists after upgrade ===\"\n ALLOW_IANA_RESTRICTED_ADDRESSES=true timeout 20 python3 ./changedetection.py -d /tmp/data > /tmp/ui-test.log 2>&1 &\n APP_PID=$!\n\n # Wait for app to be ready and fetch UI\n echo \"Waiting for current version to be ready...\"\n sleep 5\n curl --retry 6 --retry-delay 1 --retry-connrefused --silent http://127.0.0.1:5000 > /tmp/ui-output.html\n\n # Verify tag exists in UI\n if grep -q \"github-group-test\" /tmp/ui-output.html; then\n echo \"✓ Tag 'github-group-test' found in UI after upgrade\"\n else\n echo \"ERROR: Tag 'github-group-test' not found in UI after upgrade\"\n echo \"=== UI Output ===\"\n cat /tmp/ui-output.html\n echo \"=== App Log ===\"\n cat /tmp/ui-test.log\n kill $APP_PID || true\n exit 1\n fi\n\n # Verify test URL exists in UI\n if grep -q \"http://localhost/test.txt\" /tmp/ui-output.html; then\n echo \"✓ Watch URL 'http://localhost/test.txt' found in UI after upgrade\"\n else\n echo \"ERROR: Watch URL 'http://localhost/test.txt' not found in UI after upgrade\"\n echo \"=== UI Output ===\"\n cat /tmp/ui-output.html\n echo \"=== App Log ===\"\n cat /tmp/ui-test.log\n kill $APP_PID || true\n exit 1\n fi\n\n # Cleanup\n kill $APP_PID || true\n wait $APP_PID || true\n\n echo \"\"\n echo \"✓✓✓ Upgrade test passed: 0.49.1 → ${{ github.ref_name }} ✓✓✓\"\n echo \" - Commit: ${{ github.sha }}\"\n echo \" - Datastore migrated successfully\"\n echo \" - Tag 'github-group-test' survived upgrade\"\n echo \" - Watch URL 'http://localhost/test.txt' survived upgrade\"\n\n echo \"✓ Upgrade test passed: 0.49.1 → ${{ github.ref_name }}\"\n\n - name: Upload upgrade test logs\n if: always()\n uses: actions/upload-artifact@v7\n with:\n name: upgrade-test-logs-py${{ env.PYTHON_VERSION }}\n path: /tmp/upgrade-test.log\n" }, { "path": ".gitignore", "content": "# Byte-compiled / optimized / DLL files\n**/__pycache__\n**/*.py[cod]\n\n# Caches\n.mypy_cache/\n.pytest_cache/\n.ruff_cache/\n\n# Distribution / packaging\nbuild/\ndist/\n*.egg-info*\n\n# Virtual environment\n.env\n.venv/\nvenv/\n.python-version\n\n# IDEs\n.idea\n.vscode/settings.json\n*~\n\n# Datastore files\ndatastore/\ntest-datastore/\n\n# Memory consumption log\ntest-memory.log\ntests/logs/\n" }, { "path": ".pre-commit-config.yaml", "content": "repos:\n - repo: https://github.com/astral-sh/ruff-pre-commit\n rev: v0.11.2\n hooks:\n # Lint (and apply safe fixes)\n - id: ruff\n args: [--fix]\n # Fomrat\n - id: ruff-format\n" }, { "path": ".ruff.toml", "content": "# Minimum supported version\ntarget-version = \"py310\"\n\n# Formatting options\nline-length = 100\nindent-width = 4\n\nexclude = [\n \"__pycache__\",\n \".eggs\",\n \".git\",\n \".tox\",\n \".venv\",\n \"*.egg-info\",\n \"*.pyc\",\n]\n\n[lint]\n# https://docs.astral.sh/ruff/rules/\nselect = [\n \"B\", # flake8-bugbear\n \"B9\",\n \"C\", \n \"E\", # pycodestyle\n \"F\", # Pyflakes\n \"I\", # isort\n \"N\", # pep8-naming\n \"UP\", # pyupgrade\n \"W\", # pycodestyle\n]\nignore = [\n \"B007\", # unused-loop-control-variable\n \"B909\", # loop-iterator-mutation\n \"E203\", # whitespace-before-punctuation\n \"E266\", # multiple-leading-hashes-for-block-comment\n \"E501\", # redundant-backslash\n \"F403\", # undefined-local-with-import-star\n \"N802\", # invalid-function-name\n \"N806\", # non-lowercase-variable-in-function\n \"N815\", # mixed-case-variable-in-class-scope\n]\n\n[lint.mccabe]\nmax-complexity = 12\n\n[format]\nindent-style = \"space\"\nquote-style = \"preserve\"\n" }, { "path": "COMMERCIAL_LICENCE.md", "content": "# Generally\n\nIn any commercial activity involving 'Hosting' (as defined herein), whether in part or in full, this license must be executed and adhered to.\n\n# Commercial License Agreement\n\nThis Commercial License Agreement (\"Agreement\") is entered into by and between Web Technologies s.r.o. here-in (\"Licensor\") and (your company or personal name) _____________ (\"Licensee\"). This Agreement sets forth the terms and conditions under which Licensor provides its software (\"Software\") and services to Licensee for the purpose of reselling the software either in part or full, as part of any commercial activity where the activity involves a third party.\n\n### Definition of Hosting\n\nFor the purposes of this Agreement, \"hosting\" means making the functionality of the Program or modified version available to third parties as a service. This includes, without limitation:\n- Enabling third parties to interact with the functionality of the Program or modified version remotely through a computer network.\n- Offering a service the value of which entirely or primarily derives from the value of the Program or modified version.\n- Offering a service that accomplishes for users the primary purpose of the Program or modified version.\n\n## 1. Grant of License\nSubject to the terms and conditions of this Agreement, Licensor grants Licensee a non-exclusive, non-transferable license to install, use, and resell the Software. Licensee may:\n- Resell the Software as part of a service offering or as a standalone product.\n- Host the Software on a server and provide it as a hosted service (e.g., Software as a Service - SaaS).\n- Integrate the Software into a larger product or service that is then sold or provided for commercial purposes, where the software is used either in part or full.\n\n## 2. License Fees\nLicensee agrees to pay Licensor the license fees specified in the ordering document. License fees are due and payable as specified in the ordering document. The fees may include initial licensing costs and recurring fees based on the number of end users, instances of the Software resold, or revenue generated from the resale activities.\n\n## 3. Resale Conditions\nLicensee must comply with the following conditions when reselling the Software, whether the software is resold in part or full:\n- Provide end users with access to the source code under the same open-source license conditions as provided by Licensor.\n- Clearly state in all marketing and sales materials that the Software is provided under a commercial license from Licensor, and provide a link back to https://changedetection.io.\n- Ensure end users are aware of and agree to the terms of the commercial license prior to resale.\n- Do not sublicense or transfer the Software to third parties except as part of an authorized resale activity.\n\n## 4. Hosting and Provision of Services\nLicensee may host the Software (either in part or full) on its servers and provide it as a hosted service to end users. The following conditions apply:\n- Licensee must ensure that all hosted versions of the Software comply with the terms of this Agreement.\n- Licensee must provide Licensor with regular reports detailing the number of end users and instances of the hosted service.\n- Any modifications to the Software made by Licensee for hosting purposes must be made available to end users under the same open-source license conditions, unless agreed otherwise.\n\n## 5. Services\nLicensor will provide support and maintenance services as described in the support policy referenced in the ordering document should such an agreement be signed by all parties. Additional fees may apply for support services provided to end users resold by Licensee.\n\n## 6. Reporting and Audits\nLicensee agrees to provide Licensor with regular reports detailing the number of instances, end users, and revenue generated from the resale of the Software. Licensor reserves the right to audit Licensee’s records to ensure compliance with this Agreement.\n\n## 7. Term and Termination\nThis Agreement shall commence on the effective date and continue for the period set forth in the ordering document unless terminated earlier in accordance with this Agreement. Either party may terminate this Agreement if the other party breaches any material term and fails to cure such breach within thirty (30) days after receipt of written notice.\n\n## 8. Limitation of Liability and Disclaimer of Warranty\nExecuting this commercial license does not waive the Limitation of Liability or Disclaimer of Warranty as stated in the open-source LICENSE provided with the Software. The Software is provided \"as is,\" without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose, and noninfringement. In no event shall the authors or copyright holders be liable for any claim, damages, or other liability, whether in an action of contract, tort, or otherwise, arising from, out of, or in connection with the Software or the use or other dealings in the Software.\n\n## 9. Governing Law\nThis Agreement shall be governed by and construed in accordance with the laws of the Czech Republic.\n\n## Contact Information\nFor commercial licensing inquiries, please contact contact@changedetection.io and dgtlmoon@gmail.com.\n" }, { "path": "CONTRIBUTING.md", "content": "Contributing is always welcome!\n\nI am no professional flask developer, if you know a better way that something can be done, please let me know!\n\nOtherwise, it's always best to PR into the `master` branch.\n\nPlease be sure that all new functionality has a matching test!\n\nUse `pytest` to validate/test, you can run the existing tests as `pytest tests/test_notification.py` for example\n" }, { "path": "Dockerfile", "content": "# pip dependencies install stage\n\nARG PYTHON_VERSION=3.11\n\nFROM python:${PYTHON_VERSION}-slim-bookworm AS builder\n\n# See `cryptography` pin comment in requirements.txt\n\nRUN apt-get update && apt-get install -y --no-install-recommends \\\n g++ \\\n gcc \\\n libc-dev \\\n libffi-dev \\\n libjpeg-dev \\\n libssl-dev \\\n libxslt-dev \\\n make \\\n patch \\\n pkg-config \\\n zlib1g-dev\n\nRUN mkdir /install\nWORKDIR /install\n\nCOPY requirements.txt /requirements.txt\n\n# Use cache mounts and multiple wheel sources for faster ARM builds\nENV PIP_CACHE_DIR=/tmp/pip-cache\n# Help Rust find OpenSSL for cryptography package compilation on ARM\nENV PKG_CONFIG_PATH=\"/usr/lib/pkgconfig:/usr/lib/arm-linux-gnueabihf/pkgconfig:/usr/lib/aarch64-linux-gnu/pkgconfig\"\nENV PKG_CONFIG_ALLOW_SYSTEM_CFLAGS=1\nENV OPENSSL_DIR=\"/usr\"\nENV OPENSSL_LIB_DIR=\"/usr/lib/arm-linux-gnueabihf\"\nENV OPENSSL_INCLUDE_DIR=\"/usr/include/openssl\"\n# Additional environment variables for cryptography Rust build\nENV CRYPTOGRAPHY_DONT_BUILD_RUST=1\n\nRUN --mount=type=cache,id=pip,sharing=locked,target=/tmp/pip-cache \\\n pip install \\\n --prefer-binary \\\n --extra-index-url https://www.piwheels.org/simple \\\n --extra-index-url https://pypi.anaconda.org/ARM-software/simple \\\n --cache-dir=/tmp/pip-cache \\\n --target=/dependencies \\\n -r /requirements.txt\n\n# Playwright is an alternative to Selenium\n# Excluded this package from requirements.txt to prevent arm/v6 and arm/v7 builds from failing\n# https://github.com/dgtlmoon/changedetection.io/pull/1067 also musl/alpine (not supported)\nRUN --mount=type=cache,id=pip,sharing=locked,target=/tmp/pip-cache \\\n pip install \\\n --prefer-binary \\\n --cache-dir=/tmp/pip-cache \\\n --target=/dependencies \\\n playwright~=1.56.0 \\\n || echo \"WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled.\"\n\n# OpenCV is optional for fast image comparison (pixelmatch is the fallback)\n# Skip on arm/v7 and arm/v8 where builds take weeks - excluded from requirements.txt\nARG TARGETPLATFORM\nRUN --mount=type=cache,id=pip,sharing=locked,target=/tmp/pip-cache \\\n case \"$TARGETPLATFORM\" in \\\n linux/arm/v7|linux/arm/v8) \\\n echo \"INFO: Skipping OpenCV on $TARGETPLATFORM (build takes too long), using pixelmatch fallback\" \\\n ;; \\\n *) \\\n pip install \\\n --prefer-binary \\\n --extra-index-url https://www.piwheels.org/simple \\\n --cache-dir=/tmp/pip-cache \\\n --target=/dependencies \\\n opencv-python-headless>=4.8.0.76 \\\n || echo \"WARN: OpenCV install failed, will use pixelmatch fallback\" \\\n ;; \\\n esac\n\n\n# Final image stage\nFROM python:${PYTHON_VERSION}-slim-bookworm\nLABEL org.opencontainers.image.source=\"https://github.com/dgtlmoon/changedetection.io\"\nLABEL org.opencontainers.image.url=\"https://changedetection.io\"\nLABEL org.opencontainers.image.documentation=\"https://changedetection.io/tutorials\"\nLABEL org.opencontainers.image.title=\"changedetection.io\"\nLABEL org.opencontainers.image.description=\"Self-hosted web page change monitoring and notification service\"\nLABEL org.opencontainers.image.licenses=\"Apache-2.0\"\nLABEL org.opencontainers.image.vendor=\"changedetection.io\"\n\nRUN apt-get update && apt-get install -y --no-install-recommends \\\n libxslt1.1 \\\n # For presenting price amounts correctly in the restock/price detection overview\n locales \\\n # For pdftohtml\n poppler-utils \\\n # favicon type detection and other uses\n file \\\n zlib1g \\\n # OpenCV dependencies for image processing\n libglib2.0-0 \\\n libsm6 \\\n libxext6 \\\n libxrender-dev \\\n && apt-get clean && rm -rf /var/lib/apt/lists/*\n\n\n# https://stackoverflow.com/questions/58701233/docker-logs-erroneously-appears-empty-until-container-stops\nENV PYTHONUNBUFFERED=1\n\nRUN [ ! -d \"/datastore\" ] && mkdir /datastore\n\n# Re #80, sets SECLEVEL=1 in openssl.conf to allow monitoring sites with weak/old cipher suites\nRUN sed -i 's/^CipherString = .*/CipherString = DEFAULT@SECLEVEL=1/' /etc/ssl/openssl.cnf\n\n# Copy modules over to the final image and add their dir to PYTHONPATH\nCOPY --from=builder /dependencies /usr/local\nENV PYTHONPATH=/usr/local\n\nEXPOSE 5000\n\n# The actual flask app module\nCOPY changedetectionio /app/changedetectionio\n\n# Compile translation files for i18n support\nRUN pybabel compile -d /app/changedetectionio/translations\n\n# Also for OpenAPI validation wrapper - needs the YML\nRUN [ ! -d \"/app/docs\" ] && mkdir /app/docs\nCOPY docs/api-spec.yaml /app/docs/api-spec.yaml\n\n# Starting wrapper\nCOPY changedetection.py /app/changedetection.py\n\n# Github Action test purpose(test-only.yml).\n# On production, it is effectively LOGGER_LEVEL=''.\nARG LOGGER_LEVEL=''\nENV LOGGER_LEVEL=\"$LOGGER_LEVEL\"\n\n# Default\nENV LC_ALL=en_US.UTF-8\n\nWORKDIR /app\n\n# Copy and set up entrypoint script for installing extra packages\nCOPY docker-entrypoint.sh /docker-entrypoint.sh\nRUN chmod +x /docker-entrypoint.sh\n\n# Set entrypoint to handle EXTRA_PACKAGES env var\nENTRYPOINT [\"/docker-entrypoint.sh\"]\n\n# Default command (can be overridden in docker-compose.yml)\nCMD [\"python\", \"./changedetection.py\", \"-d\", \"/datastore\"]\n\n\n" }, { "path": "LICENSE", "content": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright 2025 Web Technologies s.r.o.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, { "path": "MANIFEST.in", "content": "recursive-include changedetectionio/api *\ninclude docs/api-spec.yaml\nrecursive-include changedetectionio/blueprint *\nrecursive-include changedetectionio/conditions *\nrecursive-include changedetectionio/content_fetchers *\nrecursive-include changedetectionio/jinja2_custom *\nrecursive-include changedetectionio/model *\nrecursive-include changedetectionio/notification *\nrecursive-include changedetectionio/processors *\nrecursive-include changedetectionio/realtime *\nrecursive-include changedetectionio/static *\nrecursive-include changedetectionio/store *\nrecursive-include changedetectionio/templates *\nrecursive-include changedetectionio/tests *\nrecursive-include changedetectionio/translations *\nrecursive-include changedetectionio/widgets *\nprune changedetectionio/static/package-lock.json\nprune changedetectionio/static/styles/node_modules\nprune changedetectionio/static/styles/package-lock.json\ninclude changedetectionio/favicon_utils.py\ninclude changedetection.py\ninclude requirements.txt\ninclude README-pip.md\nglobal-exclude *.pyc\nglobal-exclude node_modules\nglobal-exclude venv\n\nglobal-exclude test-datastore\nglobal-exclude changedetection.io*dist-info\nglobal-exclude changedetectionio/tests/proxy_socks5/test-datastore\n" }, { "path": "README-pip.md", "content": "# Monitor website changes\n\nDetect WebPage Changes Automatically — Monitor Web Page Changes in Real Time\n\nMonitor websites for updates — get notified via Discord, Email, Slack, Telegram, Webhook and many more.\n\nDetect web page content changes and get instant alerts.\n\n\n[Changedetection.io is the best tool to monitor web-pages for changes](https://changedetection.io) Track website content changes and receive notifications via Discord, Email, Slack, Telegram and 90+ more\n\nIdeal for monitoring price changes, content edits, conditional changes and more.\n\n[\"Self-hosted](https://changedetection.io)\n\n\n[**Don't have time? Try our extremely affordable subscription use our proxies and support!**](https://changedetection.io)\n\n\n\n### Target specific parts of the webpage using the Visual Selector tool.\n\nAvailable when connected to a playwright content fetcher (included as part of our subscription service)\n\n[\"Select](https://changedetection.io?src=pip)\n\n### Easily see what changed, examine by word, line, or individual character.\n\n[\"Self-hosted](https://changedetection.io?src=pip)\n\n\n### Perform interactive browser steps\n\nFill in text boxes, click buttons and more, setup your changedetection scenario.\n\nUsing the **Browser Steps** configuration, add basic steps before performing change detection, such as logging into websites, adding a product to a cart, accept cookie logins, entering dates and refining searches.\n\n[\"Website](https://changedetection.io?src=pip)\n\nAfter **Browser Steps** have been run, then visit the **Visual Selector** tab to refine the content you're interested in.\nRequires Playwright to be enabled.\n\n\n### Example use cases\n\n- Products and services have a change in pricing\n- _Out of stock notification_ and _Back In stock notification_\n- Monitor and track PDF file changes, know when a PDF file has text changes.\n- Governmental department updates (changes are often only on their websites)\n- New software releases, security advisories when you're not on their mailing list.\n- Festivals with changes\n- Discogs restock alerts and monitoring\n- Realestate listing changes\n- Know when your favourite whiskey is on sale, or other special deals are announced before anyone else\n- COVID related news from government websites\n- University/organisation news from their website\n- Detect and monitor changes in JSON API responses\n- JSON API monitoring and alerting\n- Changes in legal and other documents\n- Trigger API calls via notifications when text appears on a website\n- Glue together APIs using the JSON filter and JSON notifications\n- Create RSS feeds based on changes in web content\n- Monitor HTML source code for unexpected changes, strengthen your PCI compliance\n- You have a very sensitive list of URLs to watch and you do _not_ want to use the paid alternatives. (Remember, _you_ are the product)\n- Get notified when certain keywords appear in Twitter search results\n- Proactively search for jobs, get notified when companies update their careers page, search job portals for keywords.\n- Get alerts when new job positions are open on Bamboo HR and other job platforms\n- Website defacement monitoring\n- Pokémon Card Restock Tracker / Pokémon TCG Tracker\n- RegTech - stay ahead of regulatory changes, regulatory compliance\n\n_Need an actual Chrome runner with Javascript support? We support fetching via WebDriver and Playwright!_\n\n#### Key Features\n\n- Lots of trigger filters, such as \"Trigger on text\", \"Remove text by selector\", \"Ignore text\", \"Extract text\", also using regular-expressions!\n- Target elements with xPath(1.0) and CSS Selectors, Easily monitor complex JSON with JSONPath or jq\n- Switch between fast non-JS and Chrome JS based \"fetchers\"\n- Track changes in PDF files (Monitor text changed in the PDF, Also monitor PDF filesize and checksums)\n- Easily specify how often a site should be checked\n- Execute JS before extracting text (Good for logging in, see examples in the UI!)\n- Override Request Headers, Specify `POST` or `GET` and other methods\n- Use the \"Visual Selector\" to help target specific elements\n- Configurable [proxy per watch](https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration)\n- Send a screenshot with the notification when a change is detected in the web page\n\nWe [recommend and use Bright Data](https://brightdata.grsm.io/n0r16zf7eivq) global proxy services, Bright Data will match any first deposit up to $100 using our signup link.\n\n[Oxylabs](https://oxylabs.go2cloud.org/SH2d) is also an excellent proxy provider and well worth using, they offer Residential, ISP, Rotating and many other proxy types to suit your project.\n\nPlease :star: star :star: this project and help it grow! https://github.com/dgtlmoon/changedetection.io/\n\n\n\n```bash\n$ pip3 install changedetection.io\n```\n\nSpecify a target for the *datastore path* with `-d` (required) and a *listening port* with `-p` (defaults to `5000`)\n\n```bash\n$ changedetection.io -d /path/to/empty/data/dir -p 5000\n```\n\n\nThen visit http://127.0.0.1:5000 , You should now be able to access the UI.\n\nSee https://changedetection.io for more information.\n" }, { "path": "README.md", "content": "# Detect Website Changes Automatically — Monitor Web Page Changes in Real Time\n\nMonitor websites for updates — get notified via Discord, Email, Slack, Telegram, Webhook and many more.\n\n**Detect web page content changes and get instant alerts.** \n\nIdeal for monitoring price changes, content edits, conditional changes and more.\n\n\n[\"Web](https://changedetection.io?src=github)\n\n[![Release Version][release-shield]][release-link] [![Docker Pulls][docker-pulls]][docker-link] [![License][license-shield]](LICENSE.md)\n\n![changedetection.io](https://github.com/dgtlmoon/changedetection.io/actions/workflows/test-only.yml/badge.svg?branch=master)\n\n[**Get started with website page change monitoring straight away. Don't have time? Try our $8.99/month subscription, use our proxies and support!**](https://changedetection.io) , _half the price of other website change monitoring services!_\n\n\n- Chrome browser included.\n- Nothing to install, access via browser login after signup.\n- Super fast, no registration needed setup.\n- Get started watching and receiving website change notifications straight away.\n- See our [tutorials and how-to page for more inspiration](https://changedetection.io/tutorials) \n\n### Target specific parts of the webpage using the Visual Selector tool.\n\nAvailable when connected to a playwright content fetcher (included as part of our subscription service)\n\n[\"Select](https://changedetection.io?src=github)\n\n### Easily see what changed, examine by word, line, or individual character.\n\n[\"Self-hosted](https://changedetection.io?src=github)\n\n\n### Perform interactive browser steps\n\nFill in text boxes, click buttons and more, setup your changedetection scenario. \n\nUsing the **Browser Steps** configuration, add basic steps before performing change detection, such as logging into websites, adding a product to a cart, accept cookie logins, entering dates and refining searches.\n\n[\"Website](https://changedetection.io?src=github)\n\nAfter **Browser Steps** have been run, then visit the **Visual Selector** tab to refine the content you're interested in.\nRequires Playwright to be enabled.\n\n### Awesome restock and price change notifications\n\nEnable the _\"Re-stock & Price detection for single product pages\"_ option to activate the best way to monitor product pricing, this will extract any meta-data in the HTML page and give you many options to follow the pricing of the product.\n\nEasily organise and monitor prices for products from the dashboard, get alerts and notifications when the price of a product changes or comes back in stock again!\n\n[\"Easily](https://changedetection.io?src=github)\n\nSet price change notification parameters, upper and lower price, price change percentage and more.\nAlways know when a product for sale drops in price.\n\n[\"Set](https://changedetection.io?src=github)\n\n\n\n### Example use cases\n\n- Products and services have a change in pricing\n- _Out of stock notification_ and _Back In stock notification_\n- Monitor and track PDF file changes, know when a PDF file has text changes.\n- Governmental department updates (changes are often only on their websites)\n- New software releases, security advisories when you're not on their mailing list.\n- Festivals with changes\n- Discogs restock alerts and monitoring\n- Realestate listing changes\n- Know when your favourite whiskey is on sale, or other special deals are announced before anyone else\n- COVID related news from government websites\n- University/organisation news from their website\n- Detect and monitor changes in JSON API responses \n- JSON API monitoring and alerting\n- Changes in legal and other documents\n- Trigger API calls via notifications when text appears on a website\n- Glue together APIs using the JSON filter and JSON notifications\n- Create RSS feeds based on changes in web content\n- Monitor HTML source code for unexpected changes, strengthen your PCI compliance\n- You have a very sensitive list of URLs to watch and you do _not_ want to use the paid alternatives. (Remember, _you_ are the product)\n- Get notified when certain keywords appear in Twitter search results\n- Proactively search for jobs, get notified when companies update their careers page, search job portals for keywords.\n- Get alerts when new job positions are open on Bamboo HR and other job platforms\n- Website defacement monitoring\n- Pokémon Card Restock Tracker / Pokémon TCG Tracker\n- RegTech - stay ahead of regulatory changes, regulatory compliance\n\n_Need an actual Chrome runner with Javascript support? We support fetching via WebDriver and Playwright!_\n\n#### Key Features\n\n- Lots of trigger filters, such as \"Trigger on text\", \"Remove text by selector\", \"Ignore text\", \"Extract text\", also using regular-expressions!\n- Target elements with xPath 1 and xPath 2, CSS Selectors, Easily monitor complex JSON with JSONPath or jq\n- Switch between fast non-JS and Chrome JS based \"fetchers\"\n- Track changes in PDF files (Monitor text changed in the PDF, Also monitor PDF filesize and checksums)\n- Easily specify how often a site should be checked\n- Execute JS before extracting text (Good for logging in, see examples in the UI!)\n- Override Request Headers, Specify `POST` or `GET` and other methods\n- Use the \"Visual Selector\" to help target specific elements\n- Configurable [proxy per watch](https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration)\n- Send a screenshot with the notification when a change is detected in the web page\n\nWe [recommend and use Bright Data](https://brightdata.grsm.io/n0r16zf7eivq) global proxy services, Bright Data will match any first deposit up to $150 using our signup link.\n\nPlease :star: star :star: this project and help it grow! https://github.com/dgtlmoon/changedetection.io/\n\n### Conditional web page changes\n\nEasily [configure conditional actions](https://changedetection.io/tutorial/conditional-actions-web-page-changes), for example, only trigger when a price is above or below a preset amount, or [when a web page includes (or does not include) a keyword](https://changedetection.io/tutorial/how-monitor-keywords-any-website)\n\n\"Conditional\n\n### Schedule web page watches in any timezone, limit by day of week and time.\n\nEasily set a re-check schedule, for example you could limit the web page change detection to only operate during business hours.\nOr perhaps based on a foreign timezone (for example, you want to check for the latest news-headlines in a foreign country at 0900 AM),\n\n\"How\n\nIncludes quick short-cut buttons to setup a schedule for **business hours only**, or **weekends**.\n\n### We have a Chrome extension!\n\nEasily add the current web page to your changedetection.io tool, simply install the extension and click \"Sync\" to connect it to your existing changedetection.io install.\n\n[\"Chrome](https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop)\n\n[Goto the Chrome Webstore to download the extension.](https://chromewebstore.google.com/detail/changedetectionio-website/kefcfmgmlhmankjmnbijimhofdjekbop) ( Or check out the [GitHub repo](https://github.com/dgtlmoon/changedetection.io-browser-extension) ) \n\n## Installation\n\n### Docker\n\nWith Docker composer, just clone this repository and..\n\n```bash\n$ docker compose up -d\n```\n\nDocker standalone\n```bash\n$ docker run -d --restart always -p \"127.0.0.1:5000:5000\" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io\n```\n\n`:latest` tag is our latest stable release, `:dev` tag is our bleeding edge `master` branch.\n\nAlternative docker repository over at ghcr - [ghcr.io/dgtlmoon/changedetection.io](https://ghcr.io/dgtlmoon/changedetection.io)\n\n### Windows\n\nSee the install instructions at the wiki https://github.com/dgtlmoon/changedetection.io/wiki/Microsoft-Windows\n\n### Python Pip\n\nCheck out our pypi page https://pypi.org/project/changedetection.io/\n\n```bash\n$ pip3 install changedetection.io\n$ changedetection.io -d /path/to/empty/data/dir -p 5000\n```\n\nThen visit http://127.0.0.1:5000 , You should now be able to access the UI.\n\n_Now with per-site configurable support for using a fast built in HTTP fetcher or use a Chrome based fetcher for monitoring of JavaScript websites!_\n\n## Updating changedetection.io\n\n### Docker\n```\ndocker pull dgtlmoon/changedetection.io\ndocker kill $(docker ps -a -f name=changedetection.io -q)\ndocker rm $(docker ps -a -f name=changedetection.io -q)\ndocker run -d --restart always -p \"127.0.0.1:5000:5000\" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io\n```\n\n### docker compose\n\n```bash\ndocker compose pull && docker compose up -d\n```\n\nSee the wiki for more information https://github.com/dgtlmoon/changedetection.io/wiki\n\n## Different browser viewport sizes (mobile, desktop etc)\n\nIf you are using the recommended `sockpuppetbrowser` (which is in the docker-compose.yml as a setting to be uncommented) you can easily set different viewport sizes for your web page change detection, [see more information here about setting up different viewport sizes](https://github.com/dgtlmoon/sockpuppetbrowser?tab=readme-ov-file#setting-viewport-size).\n\n## Filters\n\nXPath(1.0), JSONPath, jq, and CSS support comes baked in! You can be as specific as you need, use XPath exported from various XPath element query creation tools. \n(We support LXML `re:test`, `re:match` and `re:replace`.)\n\n## Notifications\n\nChangeDetection.io supports a massive amount of notifications (including email, office365, custom APIs, etc) when a web-page has a change detected thanks to the apprise library.\nSimply set one or more notification URL's in the _[edit]_ tab of that watch.\n\nJust some examples\n\n discord://webhook_id/webhook_token\n flock://app_token/g:channel_id\n gitter://token/room\n gchat://workspace/key/token\n msteams://TokenA/TokenB/TokenC/\n o365://TenantID:AccountEmail/ClientID/ClientSecret/TargetEmail\n rocket://user:password@hostname/#Channel\n mailto://user:pass@example.com?to=receivingAddress@example.com\n json://someserver.com/custom-api\n syslog://\n \nAnd everything else in this list!\n\n\"Self-hosted\n\nNow you can also customise your notification content and use Jinja2 templating for their title and body!\n\n## JSON API Monitoring\n\nDetect changes and monitor data in JSON API's by using either JSONPath or jq to filter, parse, and restructure JSON as needed.\n\n![image](https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/json-filter-field-example.png)\n\nThis will re-parse the JSON and apply formatting to the text, making it super easy to monitor and detect changes in JSON API results\n\n![image](https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/json-diff-example.png)\n\n### JSONPath or jq?\n\nFor more complex parsing, filtering, and modifying of JSON data, jq is recommended due to the built-in operators and functions. Refer to the [documentation](https://stedolan.github.io/jq/manual/) for more specific information on jq.\n\nOne big advantage of `jq` is that you can use logic in your JSON filter, such as filters to only show items that have a value greater than/less than etc.\n\nSee the wiki https://github.com/dgtlmoon/changedetection.io/wiki/JSON-Selector-Filter-help for more information and examples\n\n### Parse JSON embedded in HTML!\n\nWhen you enable a `json:` or `jq:` filter, you can even automatically extract and parse embedded JSON inside a HTML page! Amazingly handy for sites that build content based on JSON, such as many e-commerce websites. \n\n```\n\n...\n\n``` \n\n`json:$..price` or `jq:..price` would give `3949.99`, or you can extract the whole structure (use a JSONpath test website to validate with)\n\nThe application also supports notifying you that it can follow this information automatically\n\n\n## Proxy Configuration\n\nSee the wiki https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration , we also support using [Bright Data proxy services where possible](https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#brightdata-proxy-support) and [Oxylabs](https://oxylabs.go2cloud.org/SH2d) proxy services.\n\n## Raspberry Pi support?\n\nRaspberry Pi and linux/arm/v6 linux/arm/v7 arm64 devices are supported! See the wiki for [details](https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver)\n\n## Import support\n\nEasily [import your list of websites to watch for changes in Excel .xslx file format](https://changedetection.io/tutorial/how-import-your-website-change-detection-lists-excel), or paste in lists of website URLs as plaintext. \n\nExcel import is recommended - that way you can better organise tags/groups of websites and other features.\n\n\n## API Support\n\nFull REST API for programmatic management of watches, tags, notifications and more. \n\n- **[Interactive API Documentation](https://changedetection.io/docs/api_v1/index.html)** - Complete API reference with live testing\n- **[OpenAPI Specification](docs/api-spec.yaml)** - Generate SDKs for any programming language\n\n## Support us\n\nDo you use changedetection.io to make money? does it save you time or money? Does it make your life easier? less stressful? Remember, we write this software when we should be doing actual paid work, we have to buy food and pay rent just like you.\n\n\nConsider taking out an officially supported [website change detection subscription](https://changedetection.io?src=github) , even if you don't use it, you still get the warm fuzzy feeling of helping out the project. (And who knows, you might just use it!)\n\n## Commercial Support\n\nI offer commercial support, this software is depended on by network security, aerospace , data-science and data-journalist professionals just to name a few, please reach out at dgtlmoon@gmail.com for any enquiries, I am more than glad to work with your organisation to further the possibilities of what can be done with changedetection.io\n\n\n[release-shield]: https://img.shields.io:/github/v/release/dgtlmoon/changedetection.io?style=for-the-badge\n[docker-pulls]: https://img.shields.io/docker/pulls/dgtlmoon/changedetection.io?style=for-the-badge\n[test-shield]: https://github.com/dgtlmoon/changedetection.io/actions/workflows/test-only.yml/badge.svg?branch=master\n\n[license-shield]: https://img.shields.io/github/license/dgtlmoon/changedetection.io.svg?style=for-the-badge\n[release-link]: https://github.com/dgtlmoon/changedetection.io/releases\n[docker-link]: https://hub.docker.com/r/dgtlmoon/changedetection.io\n\n## Commercial Licencing\n\nIf you are reselling this software either in part or full as part of any commercial arrangement, you must abide by our COMMERCIAL_LICENCE.md found in our code repository, please contact dgtlmoon@gmail.com and contact@changedetection.io .\n\n## Third-party licenses\n\nchangedetectionio.html_tools.elementpath_tostring: Copyright (c), 2018-2021, SISSA (Scuola Internazionale Superiore di Studi Avanzati), Licensed under [MIT license](https://github.com/sissaschool/elementpath/blob/master/LICENSE)\n\n## Contributors\n\nRecognition of fantastic contributors to the project\n\n- Constantin Hong https://github.com/Constantin1489\n" }, { "path": "babel.cfg", "content": "[python: **.py]\nkeywords = _:1,_l:1,gettext:1\n\n[jinja2: **/templates/**.html]\nencoding = utf-8\n" }, { "path": "changedetection.py", "content": "#!/usr/bin/env python3\n\n# Only exists for direct CLI usage\n\nimport changedetectionio\n\nif __name__ == '__main__':\n changedetectionio.main()\n" }, { "path": "changedetectionio/.gitignore", "content": "test-datastore\npackage-lock.json\n" }, { "path": "changedetectionio/PLUGIN_README.md", "content": "# Creating Plugins for changedetection.io\n\nThis document describes how to create plugins for changedetection.io. Plugins can be used to extend the functionality of the application in various ways.\n\n## Plugin Types\n\n### UI Stats Tab Plugins\n\nThese plugins can add content to the Stats tab in the Edit page. This is useful for adding custom statistics or visualizations about a watch.\n\n#### Creating a UI Stats Tab Plugin\n\n1. Create a Python file in a directory that will be loaded by the plugin system.\n\n2. Use the `global_hookimpl` decorator to implement the `ui_edit_stats_extras` hook:\n\n```python\nimport pluggy\nfrom loguru import logger\n\nglobal_hookimpl = pluggy.HookimplMarker(\"changedetectionio\")\n\n@global_hookimpl\ndef ui_edit_stats_extras(watch):\n \"\"\"Add custom content to the stats tab\"\"\"\n # Calculate or retrieve your stats\n my_stat = calculate_something(watch)\n \n # Return HTML content as a string\n html = f\"\"\"\n
\n

My Plugin Statistics

\n

My statistic: {my_stat}

\n
\n \"\"\"\n return html\n```\n\n3. The HTML you return will be included in the Stats tab.\n\n## Plugin Loading\n\nPlugins can be loaded from:\n\n1. Built-in plugin directories in the codebase\n2. External packages using setuptools entry points\n\nTo add a new plugin directory, modify the `plugin_dirs` dictionary in `pluggy_interface.py`.\n\n## Example Plugin\n\nHere's a simple example of a plugin that adds a word count statistic to the Stats tab:\n\n```python\nimport pluggy\nfrom loguru import logger\n\nglobal_hookimpl = pluggy.HookimplMarker(\"changedetectionio\")\n\ndef count_words_in_history(watch):\n \"\"\"Count words in the latest snapshot\"\"\"\n try:\n if not watch.history.keys():\n return 0\n \n latest_key = list(watch.history.keys())[-1]\n latest_content = watch.get_history_snapshot(timestamp=latest_key)\n return len(latest_content.split())\n except Exception as e:\n logger.error(f\"Error counting words: {str(e)}\")\n return 0\n\n@global_hookimpl\ndef ui_edit_stats_extras(watch):\n \"\"\"Add word count to the Stats tab\"\"\"\n word_count = count_words_in_history(watch)\n \n html = f\"\"\"\n
\n

Content Analysis

\n \n \n \n \n \n \n \n
Word count (latest snapshot){word_count}
\n
\n \"\"\"\n return html\n```\n\n## Testing Your Plugin\n\n1. Place your plugin in one of the directories scanned by the plugin system\n2. Restart changedetection.io\n3. Go to the Edit page of a watch and check the Stats tab to see your content" }, { "path": "changedetectionio/__init__.py", "content": "#!/usr/bin/env python3\n\n# Read more https://github.com/dgtlmoon/changedetection.io/wiki\n# Semver means never use .01, or 00. Should be .1.\n__version__ = '0.54.6'\n\nfrom changedetectionio.strtobool import strtobool\nfrom json.decoder import JSONDecodeError\n\nfrom loguru import logger\nimport getopt\nimport logging\nimport os\nimport platform\nimport signal\nimport threading\nimport time\n\n# Eventlet completely removed - using threading mode for SocketIO\n# This provides better Python 3.12+ compatibility and eliminates eventlet/asyncio conflicts\n# Note: store and changedetection_app are imported inside main() to avoid\n# initialization before argument parsing (allows --help to work without loading everything)\n\n# ==============================================================================\n# Multiprocessing Configuration - CRITICAL for Thread Safety\n# ==============================================================================\n#\n# PROBLEM: Python 3.12+ warns about fork() with multi-threaded processes:\n# \"This process is multi-threaded, use of fork() may lead to deadlocks\"\n#\n# WHY IT'S DANGEROUS:\n# 1. This Flask app has multiple threads (HTTP handlers, workers, SocketIO)\n# 2. fork() copies ONLY the calling thread to the child process\n# 3. BUT fork() also copies all locks/mutexes in their current state\n# 4. If another thread held a lock during fork() → child has locked lock with no owner\n# 5. Result: PERMANENT DEADLOCK if child tries to acquire that lock\n#\n# SOLUTION: Use 'spawn' instead of 'fork'\n# - spawn starts a fresh Python interpreter (no inherited threads or locks)\n# - Slower (~200ms vs ~1ms) but safe with multi-threaded parent\n# - Consistent across all platforms (Windows already uses spawn by default)\n#\n# IMPLEMENTATION:\n# 1. Explicit contexts everywhere (primary protection):\n# - playwright.py: ctx = multiprocessing.get_context('spawn')\n# - puppeteer.py: ctx = multiprocessing.get_context('spawn')\n# - isolated_opencv.py: ctx = multiprocessing.get_context('spawn')\n# - isolated_libvips.py: ctx = multiprocessing.get_context('spawn')\n#\n# 2. Global default (defense-in-depth, below):\n# - Safety net if future code forgets explicit context\n# - Protects against third-party libraries using Process()\n# - Costs nothing (explicit contexts always override it)\n#\n# WHY BOTH?\n# - Explicit contexts: Clear, self-documenting, always works\n# - Global default: Safety net for forgotten contexts or library code\n# - If someone writes \"Process()\" instead of \"ctx.Process()\", still safe!\n#\n# See: https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods\n# ==============================================================================\n\nimport multiprocessing\nimport os\nimport sys\n\n# Limit glibc malloc arena count to prevent RSS growth from concurrent requests.\n# Default: glibc creates up to 8×CPU_cores arenas. Each concurrent thread/connection\n# can trigger a new arena, and freed memory stays mapped in those arenas as RSS forever.\n# With MALLOC_ARENA_MAX=2, at most 2 arenas are used; freed pages return to the OS faster.\n# Must be set before worker threads start; env var is read lazily by glibc on first arena creation.\nif 'MALLOC_ARENA_MAX' not in os.environ:\n os.environ['MALLOC_ARENA_MAX'] = '2'\n try:\n import ctypes as _ctypes\n _ctypes.CDLL('libc.so.6').mallopt(-8, 2) # M_ARENA_MAX = -8\n except Exception:\n pass\n\n# Set spawn as global default (safety net - all our code uses explicit contexts anyway)\n# Skip in tests to avoid breaking pytest-flask's LiveServer fixture (uses unpicklable local functions)\nif 'pytest' not in sys.modules:\n try:\n if multiprocessing.get_start_method(allow_none=True) is None:\n multiprocessing.set_start_method('spawn', force=False)\n logger.debug(\"Set multiprocessing default to 'spawn' for thread safety (explicit contexts used everywhere)\")\n except RuntimeError:\n logger.debug(f\"Multiprocessing start method already set: {multiprocessing.get_start_method()}\")\n\n# Only global so we can access it in the signal handler\napp = None\ndatastore = None\n\ndef get_version():\n return __version__\n\n# Parent wrapper or OS sends us a SIGTERM/SIGINT, do everything required for a clean shutdown\ndef sigshutdown_handler(_signo, _stack_frame):\n name = signal.Signals(_signo).name\n logger.critical(f'Shutdown: Got Signal - {name} ({_signo}), Fast shutdown initiated')\n\n # Set exit flag immediately to stop all loops\n app.config.exit.set()\n datastore.stop_thread = True\n\n # Log memory consumption before shutting down workers (cross-platform)\n try:\n import psutil\n process = psutil.Process()\n mem_info = process.memory_info()\n rss_mb = mem_info.rss / 1024 / 1024\n vms_mb = mem_info.vms / 1024 / 1024\n logger.info(f\"Memory consumption before worker shutdown: RSS={rss_mb:,.2f} MB, VMS={vms_mb:,.2f} MB\")\n except Exception as e:\n logger.warning(f\"Could not retrieve memory stats: {str(e)}\")\n\n # Shutdown workers and queues immediately\n try:\n from changedetectionio import worker_pool\n worker_pool.shutdown_workers()\n except Exception as e:\n logger.error(f\"Error shutting down workers: {str(e)}\")\n \n # Close janus queues properly\n try:\n from changedetectionio.flask_app import update_q, notification_q\n update_q.close()\n notification_q.close()\n logger.debug(\"Queues closed successfully\")\n except Exception as e:\n logger.critical(f\"CRITICAL: Failed to close queues: {e}\")\n \n # Shutdown socketio server fast\n from changedetectionio.flask_app import socketio_server\n if socketio_server and hasattr(socketio_server, 'shutdown'):\n try:\n socketio_server.shutdown()\n except Exception as e:\n logger.error(f\"Error shutting down Socket.IO server: {str(e)}\")\n \n # With immediate persistence, all data is already saved\n logger.success('All data already persisted (immediate commits enabled).')\n\n sys.exit()\n\ndef print_help():\n \"\"\"Print help text for command line options\"\"\"\n print('Usage: changedetection.py [options]')\n print('')\n print('Standard options:')\n print(' -s SSL enable')\n print(' -h HOST Listen host (default: 0.0.0.0)')\n print(' -p PORT Listen port (default: 5000)')\n print(' -d PATH Datastore path')\n print(' -l LEVEL Log level (TRACE, DEBUG, INFO, SUCCESS, WARNING, ERROR, CRITICAL)')\n print(' -c Cleanup unused snapshots')\n print(' -C Create datastore directory if it doesn\\'t exist')\n print(' -P true/false Set all watches paused (true) or active (false)')\n print('')\n print('Add URLs on startup:')\n print(' -u URL Add URL to watch (can be used multiple times)')\n print(' -u0 \\'JSON\\' Set options for first -u URL (e.g. \\'{\"processor\":\"text_json_diff\"}\\')')\n print(' -u1 \\'JSON\\' Set options for second -u URL (0-indexed)')\n print(' -u2 \\'JSON\\' Set options for third -u URL, etc.')\n print(' Available options: processor, fetch_backend, headers, method, etc.')\n print(' See model/Watch.py for all available options')\n print('')\n print('Recheck on startup:')\n print(' -r all Queue all watches for recheck on startup')\n print(' -r UUID,... Queue specific watches (comma-separated UUIDs)')\n print(' -r all N Queue all watches, wait for completion, repeat N times')\n print(' -r UUID,... N Queue specific watches, wait for completion, repeat N times')\n print('')\n print('Batch mode:')\n print(' -b Run in batch mode (process queue then exit)')\n print(' Useful for CI/CD, cron jobs, or one-time checks')\n print(' NOTE: Batch mode checks if Flask is running and aborts if port is in use')\n print(' Use -p PORT to specify a different port if needed')\n print('')\n\ndef main():\n global datastore\n global app\n\n # Early help/version check before any initialization\n if '--help' in sys.argv or '-help' in sys.argv:\n print_help()\n sys.exit(0)\n\n if '--version' in sys.argv or '-v' in sys.argv:\n print(f'changedetection.io {__version__}')\n sys.exit(0)\n\n # Import heavy modules after help/version checks to keep startup fast for those flags\n from changedetectionio import store\n from changedetectionio.flask_app import changedetection_app\n\n datastore_path = None\n # Set a default logger level\n logger_level = 'DEBUG'\n include_default_watches = True\n all_paused = None # None means don't change, True/False to set\n\n host = os.environ.get(\"LISTEN_HOST\", \"0.0.0.0\").strip()\n port = int(os.environ.get('PORT', 5000))\n ssl_mode = False\n\n # Lists for multiple URLs and their options\n urls_to_add = []\n url_options = {} # Key: index (0-based), Value: dict of options\n recheck_watches = None # None, 'all', or list of UUIDs\n recheck_repeat_count = 1 # Number of times to repeat recheck cycle\n batch_mode = False # Run once then exit when queue is empty\n\n # On Windows, create and use a default path.\n if os.name == 'nt':\n datastore_path = os.path.expandvars(r'%APPDATA%\\changedetection.io')\n os.makedirs(datastore_path, exist_ok=True)\n else:\n # Must be absolute so that send_from_directory doesnt try to make it relative to backend/\n datastore_path = os.path.join(os.getcwd(), \"../datastore\")\n\n # Pre-process arguments to extract -u, -u, and -r options before getopt\n # This allows unlimited -u0, -u1, -u2, ... options without predefining them\n cleaned_argv = ['changedetection.py'] # Start with program name\n i = 1\n while i < len(sys.argv):\n arg = sys.argv[i]\n\n # Handle -u (add URL)\n if arg == '-u' and i + 1 < len(sys.argv):\n urls_to_add.append(sys.argv[i + 1])\n i += 2\n continue\n\n # Handle -u (set options for URL at index N)\n if arg.startswith('-u') and len(arg) > 2 and arg[2:].isdigit():\n idx = int(arg[2:])\n if i + 1 < len(sys.argv):\n try:\n import json\n url_options[idx] = json.loads(sys.argv[i + 1])\n except json.JSONDecodeError as e:\n print(f'Error: Invalid JSON for {arg}: {sys.argv[i + 1]}')\n print(f'JSON decode error: {e}')\n sys.exit(2)\n i += 2\n continue\n\n # Handle -r (recheck watches)\n if arg == '-r' and i + 1 < len(sys.argv):\n recheck_arg = sys.argv[i + 1]\n if recheck_arg.lower() == 'all':\n recheck_watches = 'all'\n else:\n # Parse comma-separated list of UUIDs\n recheck_watches = [uuid.strip() for uuid in recheck_arg.split(',') if uuid.strip()]\n\n # Check for optional repeat count as third argument\n if i + 2 < len(sys.argv) and sys.argv[i + 2].isdigit():\n recheck_repeat_count = int(sys.argv[i + 2])\n if recheck_repeat_count < 1:\n print(f'Error: Repeat count must be at least 1, got {recheck_repeat_count}')\n sys.exit(2)\n i += 3\n else:\n i += 2\n continue\n\n # Handle -b (batch mode - run once and exit)\n if arg == '-b':\n batch_mode = True\n i += 1\n continue\n\n # Keep other arguments for getopt\n cleaned_argv.append(arg)\n i += 1\n\n try:\n opts, args = getopt.getopt(cleaned_argv[1:], \"6Csd:h:p:l:P:\", \"port\")\n except getopt.GetoptError as e:\n print_help()\n print(f'Error: {e}')\n sys.exit(2)\n\n create_datastore_dir = False\n\n # Set a logger level via shell env variable\n # Used: Dockerfile for CICD\n # To set logger level for pytest, see the app function in tests/conftest.py\n if os.getenv(\"LOGGER_LEVEL\"):\n level = os.getenv(\"LOGGER_LEVEL\")\n logger_level = int(level) if level.isdigit() else level.upper()\n\n for opt, arg in opts:\n if opt == '-s':\n ssl_mode = True\n\n if opt == '-h':\n host = arg\n\n if opt == '-p':\n port = int(arg)\n\n if opt == '-d':\n datastore_path = arg\n\n # Create the datadir if it doesnt exist\n if opt == '-C':\n create_datastore_dir = True\n\n if opt == '-l':\n logger_level = int(arg) if arg.isdigit() else arg.upper()\n\n if opt == '-P':\n try:\n all_paused = bool(strtobool(arg))\n except ValueError:\n print(f'Error: Invalid value for -P option: {arg}')\n print('Expected: true, false, yes, no, 1, or 0')\n sys.exit(2)\n\n # If URLs are provided, don't include default watches\n if urls_to_add:\n include_default_watches = False\n\n\n logger.success(f\"changedetection.io version {get_version()} starting.\")\n # Launch using SocketIO run method for proper integration (if enabled)\n ssl_cert_file = os.getenv(\"SSL_CERT_FILE\", 'cert.pem')\n ssl_privkey_file = os.getenv(\"SSL_PRIVKEY_FILE\", 'privkey.pem')\n if os.getenv(\"SSL_CERT_FILE\") and os.getenv(\"SSL_PRIVKEY_FILE\"):\n ssl_mode = True\n\n # SSL mode could have been set by -s too, therefor fallback to default values\n if ssl_mode:\n if not os.path.isfile(ssl_cert_file) or not os.path.isfile(ssl_privkey_file):\n logger.critical(f\"Cannot start SSL/HTTPS mode, Please be sure that {ssl_cert_file}' and '{ssl_privkey_file}' exist in in {os.getcwd()}\")\n os._exit(2)\n\n # Without this, a logger will be duplicated\n logger.remove()\n try:\n log_level_for_stdout = { 'TRACE', 'DEBUG', 'INFO', 'SUCCESS' }\n logger.configure(handlers=[\n {\"sink\": sys.stdout, \"level\": logger_level,\n \"filter\" : lambda record: record['level'].name in log_level_for_stdout},\n {\"sink\": sys.stderr, \"level\": logger_level,\n \"filter\": lambda record: record['level'].name not in log_level_for_stdout},\n ])\n # Catch negative number or wrong log level name\n except ValueError:\n print(\"Available log level names: TRACE, DEBUG(default), INFO, SUCCESS,\"\n \" WARNING, ERROR, CRITICAL\")\n sys.exit(2)\n\n # Disable verbose pyppeteer logging to prevent memory leaks from large CDP messages\n # Set both parent and child loggers since pyppeteer hardcodes DEBUG level\n logging.getLogger('pyppeteer.connection').setLevel(logging.WARNING)\n logging.getLogger('pyppeteer.connection.Connection').setLevel(logging.WARNING)\n\n # isnt there some @thingy to attach to each route to tell it, that this route needs a datastore\n app_config = {\n 'datastore_path': datastore_path,\n 'batch_mode': batch_mode,\n 'recheck_watches': recheck_watches,\n 'recheck_repeat_count': recheck_repeat_count\n }\n\n if not os.path.isdir(app_config['datastore_path']):\n if create_datastore_dir:\n os.makedirs(app_config['datastore_path'], exist_ok=True)\n else:\n logger.critical(\n f\"ERROR: Directory path for the datastore '{app_config['datastore_path']}'\"\n f\" does not exist, cannot start, please make sure the\"\n f\" directory exists or specify a directory with the -d option.\\n\"\n f\"Or use the -C parameter to create the directory.\")\n sys.exit(2)\n\n try:\n datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], version_tag=__version__, include_default_watches=include_default_watches)\n except JSONDecodeError as e:\n # Dont' start if the JSON DB looks corrupt\n logger.critical(f\"ERROR: JSON DB or Proxy List JSON at '{app_config['datastore_path']}' appears to be corrupt, aborting.\")\n logger.critical(str(e))\n sys.exit(1)\n\n # Testing mode: Exit cleanly after datastore initialization (for CI/CD upgrade tests)\n if os.environ.get('TESTING_SHUTDOWN_AFTER_DATASTORE_LOAD'):\n logger.success(f\"TESTING MODE: Datastore loaded successfully from {app_config['datastore_path']}\")\n logger.success(f\"TESTING MODE: Schema version: {datastore.data['settings']['application'].get('schema_version', 'unknown')}\")\n logger.success(f\"TESTING MODE: Loaded {len(datastore.data['watching'])} watches\")\n logger.success(\"TESTING MODE: Exiting cleanly (TESTING_SHUTDOWN_AFTER_DATASTORE_LOAD is set)\")\n sys.exit(0)\n\n # Apply all_paused setting if specified via CLI\n if all_paused is not None:\n datastore.data['settings']['application']['all_paused'] = all_paused\n logger.info(f\"Setting all watches paused: {all_paused}\")\n\n # Inject datastore into plugins that need access to settings\n from changedetectionio.pluggy_interface import inject_datastore_into_plugins\n inject_datastore_into_plugins(datastore)\n\n # Step 1: Add URLs with their options (if provided via -u flags)\n added_watch_uuids = []\n if urls_to_add:\n logger.info(f\"Adding {len(urls_to_add)} URL(s) from command line\")\n for idx, url in enumerate(urls_to_add):\n extras = url_options.get(idx, {})\n if extras:\n logger.debug(f\"Adding watch {idx}: {url} with options: {extras}\")\n else:\n logger.debug(f\"Adding watch {idx}: {url}\")\n\n new_uuid = datastore.add_watch(url=url, extras=extras)\n if new_uuid:\n added_watch_uuids.append(new_uuid)\n logger.success(f\"Added watch: {url} (UUID: {new_uuid})\")\n else:\n logger.error(f\"Failed to add watch: {url}\")\n\n app = changedetection_app(app_config, datastore)\n\n # Step 2: Queue newly added watches (if -u was provided in batch mode)\n # This must happen AFTER app initialization so update_q is available\n if batch_mode and added_watch_uuids:\n from changedetectionio.flask_app import update_q\n from changedetectionio import queuedWatchMetaData, worker_pool\n\n logger.info(f\"Batch mode: Queuing {len(added_watch_uuids)} newly added watches\")\n for watch_uuid in added_watch_uuids:\n try:\n worker_pool.queue_item_async_safe(\n update_q,\n queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid})\n )\n logger.debug(f\"Queued newly added watch: {watch_uuid}\")\n except Exception as e:\n logger.error(f\"Failed to queue watch {watch_uuid}: {e}\")\n\n # Step 3: Queue watches for recheck (if -r was provided)\n # This must happen AFTER app initialization so update_q is available\n if recheck_watches is not None:\n from changedetectionio.flask_app import update_q\n from changedetectionio import queuedWatchMetaData, worker_pool\n\n watches_to_queue = []\n if recheck_watches == 'all':\n # Queue all watches, excluding those already queued in batch mode\n all_watches = list(datastore.data['watching'].keys())\n if batch_mode and added_watch_uuids:\n # Exclude newly added watches that were already queued in batch mode\n watches_to_queue = [uuid for uuid in all_watches if uuid not in added_watch_uuids]\n logger.info(f\"Queuing {len(watches_to_queue)} existing watches for recheck ({len(added_watch_uuids)} newly added watches already queued)\")\n else:\n watches_to_queue = all_watches\n logger.info(f\"Queuing all {len(watches_to_queue)} watches for recheck\")\n else:\n # Queue specific UUIDs\n watches_to_queue = recheck_watches\n logger.info(f\"Queuing {len(watches_to_queue)} specific watches for recheck\")\n\n queued_count = 0\n for watch_uuid in watches_to_queue:\n if watch_uuid in datastore.data['watching']:\n try:\n worker_pool.queue_item_async_safe(\n update_q,\n queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid})\n )\n queued_count += 1\n logger.debug(f\"Queued watch for recheck: {watch_uuid}\")\n except Exception as e:\n logger.error(f\"Failed to queue watch {watch_uuid}: {e}\")\n else:\n logger.warning(f\"Watch UUID not found in datastore: {watch_uuid}\")\n\n logger.success(f\"Successfully queued {queued_count} watches for recheck\")\n\n # Step 4: Setup batch mode monitor (if -b was provided)\n if batch_mode:\n from changedetectionio.flask_app import update_q\n\n # Safety check: Ensure Flask app is not already running on this port\n # Batch mode should never run alongside the web server\n import socket\n test_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)\n\n try:\n # Try to bind to the configured host:port (no SO_REUSEADDR - strict check)\n test_socket.bind((host, port))\n test_socket.close()\n logger.debug(f\"Batch mode: Port {port} is available (Flask app not running)\")\n except OSError as e:\n test_socket.close()\n # errno 98 = EADDRINUSE (Linux)\n # errno 48 = EADDRINUSE (macOS)\n # errno 10048 = WSAEADDRINUSE (Windows)\n if e.errno in (48, 98, 10048) or \"Address already in use\" in str(e) or \"already in use\" in str(e).lower():\n logger.critical(f\"ERROR: Batch mode cannot run - port {port} is already in use\")\n logger.critical(f\"The Flask web server appears to be running on {host}:{port}\")\n logger.critical(f\"Batch mode is designed for standalone operation (CI/CD, cron jobs, etc.)\")\n logger.critical(f\"Please either stop the Flask web server, or use a different port with -p PORT\")\n sys.exit(1)\n else:\n # Some other socket error - log but continue (might be network configuration issue)\n logger.warning(f\"Port availability check failed with unexpected error: {e}\")\n logger.warning(f\"Continuing with batch mode anyway - be aware of potential conflicts\")\n\n def queue_watches_for_recheck(datastore, iteration):\n \"\"\"Helper function to queue watches for recheck\"\"\"\n watches_to_queue = []\n if recheck_watches == 'all':\n all_watches = list(datastore.data['watching'].keys())\n if batch_mode and added_watch_uuids and iteration == 1:\n # Only exclude newly added watches on first iteration\n watches_to_queue = [uuid for uuid in all_watches if uuid not in added_watch_uuids]\n else:\n watches_to_queue = all_watches\n logger.info(f\"Batch mode (iteration {iteration}): Queuing all {len(watches_to_queue)} watches\")\n elif recheck_watches:\n watches_to_queue = recheck_watches\n logger.info(f\"Batch mode (iteration {iteration}): Queuing {len(watches_to_queue)} specific watches\")\n\n queued_count = 0\n for watch_uuid in watches_to_queue:\n if watch_uuid in datastore.data['watching']:\n try:\n worker_pool.queue_item_async_safe(\n update_q,\n queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid})\n )\n queued_count += 1\n except Exception as e:\n logger.error(f\"Failed to queue watch {watch_uuid}: {e}\")\n else:\n logger.warning(f\"Watch UUID not found in datastore: {watch_uuid}\")\n logger.success(f\"Batch mode (iteration {iteration}): Successfully queued {queued_count} watches\")\n return queued_count\n\n def batch_mode_monitor():\n \"\"\"Monitor queue and workers, shutdown or repeat when work is complete\"\"\"\n import time\n\n # Track iterations if repeat mode is enabled\n current_iteration = 1\n total_iterations = recheck_repeat_count if recheck_watches and recheck_repeat_count > 1 else 1\n\n if total_iterations > 1:\n logger.info(f\"Batch mode: Will repeat recheck {total_iterations} times\")\n else:\n logger.info(\"Batch mode: Waiting for all queued items to complete...\")\n\n # Wait a bit for workers to start processing\n time.sleep(3)\n\n try:\n while current_iteration <= total_iterations:\n logger.info(f\"Batch mode: Waiting for iteration {current_iteration}/{total_iterations} to complete...\")\n\n # Use the shared wait_for_all_checks function\n completed = worker_pool.wait_for_all_checks(update_q, timeout=300)\n\n if not completed:\n logger.warning(f\"Batch mode: Iteration {current_iteration} timed out after 300 seconds\")\n\n logger.success(f\"Batch mode: Iteration {current_iteration}/{total_iterations} completed\")\n\n # Check if we need to repeat\n if current_iteration < total_iterations:\n logger.info(f\"Batch mode: Starting iteration {current_iteration + 1}...\")\n current_iteration += 1\n\n # Re-queue watches for next iteration\n queue_watches_for_recheck(datastore, current_iteration)\n\n # Brief pause before continuing\n time.sleep(2)\n else:\n # All iterations complete\n logger.success(f\"Batch mode: All {total_iterations} iterations completed, initiating shutdown\")\n # Trigger shutdown\n import os, signal\n os.kill(os.getpid(), signal.SIGTERM)\n return\n\n except Exception as e:\n logger.error(f\"Batch mode monitor error: {e}\")\n logger.error(f\"Initiating emergency shutdown\")\n import os, signal\n os.kill(os.getpid(), signal.SIGTERM)\n\n # Start monitor in background thread\n monitor_thread = threading.Thread(target=batch_mode_monitor, daemon=True, name=\"BatchModeMonitor\")\n monitor_thread.start()\n logger.info(\"Batch mode enabled: Will exit after all queued items are processed\")\n\n # Get the SocketIO instance from the Flask app (created in flask_app.py)\n from changedetectionio.flask_app import socketio_server\n global socketio\n socketio = socketio_server\n\n signal.signal(signal.SIGTERM, sigshutdown_handler)\n signal.signal(signal.SIGINT, sigshutdown_handler)\n \n # Custom signal handler for memory cleanup\n def sigusr_clean_handler(_signo, _stack_frame):\n from changedetectionio.gc_cleanup import memory_cleanup\n logger.info('SIGUSR1 received: Running memory cleanup')\n return memory_cleanup(app)\n\n # Register the SIGUSR1 signal handler\n # Only register the signal handler if running on Linux\n if platform.system() == \"Linux\":\n signal.signal(signal.SIGUSR1, sigusr_clean_handler)\n else:\n logger.info(\"SIGUSR1 handler only registered on Linux, skipped.\")\n\n app.config['datastore_path'] = datastore_path\n\n\n @app.context_processor\n def inject_template_globals():\n return dict(right_sticky=\"v\"+__version__,\n new_version_available=app.config['NEW_VERSION_AVAILABLE'],\n has_password=datastore.data['settings']['application']['password'] != False,\n socket_io_enabled=datastore.data['settings']['application'].get('ui', {}).get('socket_io_enabled', True),\n all_paused=datastore.data['settings']['application'].get('all_paused', False),\n all_muted=datastore.data['settings']['application'].get('all_muted', False)\n )\n\n # Monitored websites will not receive a Referer header when a user clicks on an outgoing link.\n @app.after_request\n def hide_referrer(response):\n if strtobool(os.getenv(\"HIDE_REFERER\", 'false')):\n response.headers[\"Referrer-Policy\"] = \"same-origin\"\n\n return response\n\n # Proxy sub-directory support\n # Set environment var USE_X_SETTINGS=1 on this script\n # And then in your proxy_pass settings\n #\n # proxy_set_header Host \"localhost\";\n # proxy_set_header X-Forwarded-Prefix /app;\n\n\n if os.getenv('USE_X_SETTINGS'):\n logger.info(\"USE_X_SETTINGS is ENABLED\")\n from werkzeug.middleware.proxy_fix import ProxyFix\n app.wsgi_app = ProxyFix(\n app.wsgi_app,\n x_for=1, # X-Forwarded-For (client IP)\n x_proto=1, # X-Forwarded-Proto (http/https)\n x_host=1, # X-Forwarded-Host (original host)\n x_port=1, # X-Forwarded-Port (original port)\n x_prefix=1 # X-Forwarded-Prefix (URL prefix)\n )\n\n\n # In batch mode, skip starting the HTTP server - just keep workers running\n if batch_mode:\n logger.info(\"Batch mode: Skipping HTTP server startup, workers will process queue\")\n logger.info(\"Batch mode: Main thread will wait for shutdown signal\")\n # Keep main thread alive until batch monitor triggers shutdown\n try:\n while True:\n time.sleep(1)\n except KeyboardInterrupt:\n logger.info(\"Batch mode: Keyboard interrupt received\")\n pass\n else:\n # Normal mode: Start HTTP server\n # SocketIO instance is already initialized in flask_app.py\n if socketio_server:\n if ssl_mode:\n logger.success(f\"SSL mode enabled, attempting to start with '{ssl_cert_file}' and '{ssl_privkey_file}' in {os.getcwd()}\")\n socketio.run(app, host=host, port=int(port), debug=False,\n ssl_context=(ssl_cert_file, ssl_privkey_file), allow_unsafe_werkzeug=True)\n else:\n socketio.run(app, host=host, port=int(port), debug=False, allow_unsafe_werkzeug=True)\n else:\n # Run Flask app without Socket.IO if disabled\n logger.info(\"Starting Flask app without Socket.IO server\")\n if ssl_mode:\n logger.success(f\"SSL mode enabled, attempting to start with '{ssl_cert_file}' and '{ssl_privkey_file}' in {os.getcwd()}\")\n app.run(host=host, port=int(port), debug=False,\n ssl_context=(ssl_cert_file, ssl_privkey_file))\n else:\n app.run(host=host, port=int(port), debug=False)\n" }, { "path": "changedetectionio/api/Import.py", "content": "from changedetectionio.strtobool import strtobool\nfrom flask_restful import abort, Resource\nfrom flask import request\nfrom functools import wraps\nfrom . import auth, validate_openapi_request\nfrom ..validate_url import is_safe_valid_url\nimport json\n\n# Number of URLs above which import switches to background processing\nIMPORT_SWITCH_TO_BACKGROUND_THRESHOLD = 20\n\n\ndef default_content_type(content_type='text/plain'):\n \"\"\"Decorator to set a default Content-Type header if none is provided.\"\"\"\n def decorator(f):\n @wraps(f)\n def wrapper(*args, **kwargs):\n if not request.content_type:\n # Set default content type in the request environment\n request.environ['CONTENT_TYPE'] = content_type\n return f(*args, **kwargs)\n return wrapper\n return decorator\n\n\ndef convert_query_param_to_type(value, schema_property):\n \"\"\"\n Convert a query parameter string to the appropriate type based on schema definition.\n\n Args:\n value: String value from query parameter\n schema_property: Schema property definition with 'type' or 'anyOf' field\n\n Returns:\n Converted value in the appropriate type\n\n Supports both OpenAPI 3.1 formats:\n - type: [string, 'null'] (array format)\n - anyOf: [{type: string}, {type: null}] (anyOf format)\n \"\"\"\n prop_type = schema_property.get('type')\n\n # Handle OpenAPI 3.1 type arrays: type: [string, 'null']\n if isinstance(prop_type, list):\n # Use the first non-null type from the array\n for t in prop_type:\n if t != 'null':\n prop_type = t\n break\n else:\n prop_type = None\n\n # Handle anyOf schemas (older format)\n elif 'anyOf' in schema_property:\n # Use the first non-null type from anyOf\n for option in schema_property['anyOf']:\n if option.get('type') and option.get('type') != 'null':\n prop_type = option.get('type')\n break\n else:\n prop_type = None\n\n # Handle array type (e.g., notification_urls)\n if prop_type == 'array':\n # Support both comma-separated and JSON array format\n if value.startswith('['):\n try:\n return json.loads(value)\n except json.JSONDecodeError:\n return [v.strip() for v in value.split(',')]\n return [v.strip() for v in value.split(',')]\n\n # Handle object type (e.g., time_between_check, headers)\n elif prop_type == 'object':\n try:\n return json.loads(value)\n except json.JSONDecodeError:\n raise ValueError(f\"Invalid JSON object for field: {value}\")\n\n # Handle boolean type\n elif prop_type == 'boolean':\n return strtobool(value)\n\n # Handle integer type\n elif prop_type == 'integer':\n return int(value)\n\n # Handle number type (float)\n elif prop_type == 'number':\n return float(value)\n\n # Default: return as string\n return value\n\n\nclass Import(Resource):\n def __init__(self, **kwargs):\n # datastore is a black box dependency\n self.datastore = kwargs['datastore']\n\n @auth.check_token\n @default_content_type('text/plain') #3547 #3542\n @validate_openapi_request('importWatches')\n def post(self):\n \"\"\"Import a list of watched URLs with optional watch configuration.\"\"\"\n from . import get_watch_schema_properties\n # Special parameters that are NOT watch configuration\n special_params = {'tag', 'tag_uuids', 'dedupe', 'proxy'}\n\n extras = {}\n\n # Handle special 'proxy' parameter\n if request.args.get('proxy'):\n plist = self.datastore.proxy_list\n if not request.args.get('proxy') in plist:\n proxy_list_str = ', '.join(plist) if plist else 'none configured'\n return f\"Invalid proxy choice, currently supported proxies are '{proxy_list_str}'\", 400\n else:\n extras['proxy'] = request.args.get('proxy')\n\n # Handle special 'dedupe' parameter\n dedupe = strtobool(request.args.get('dedupe', 'true'))\n\n # Handle special 'tag' and 'tag_uuids' parameters\n tags = request.args.get('tag')\n tag_uuids = request.args.get('tag_uuids')\n\n if tag_uuids:\n tag_uuids = tag_uuids.split(',')\n\n # Extract ALL other query parameters as watch configuration\n # Get schema from OpenAPI spec (replaces old schema_create_watch)\n schema_properties = get_watch_schema_properties()\n for param_name, param_value in request.args.items():\n # Skip special parameters\n if param_name in special_params:\n continue\n\n # Skip if not in schema (unknown parameter)\n if param_name not in schema_properties:\n return f\"Unknown watch configuration parameter: {param_name}\", 400\n\n # Convert to appropriate type based on schema\n try:\n converted_value = convert_query_param_to_type(param_value, schema_properties[param_name])\n extras[param_name] = converted_value\n except (ValueError, json.JSONDecodeError) as e:\n return f\"Invalid value for parameter '{param_name}': {str(e)}\", 400\n\n # Validate processor if provided\n if 'processor' in extras:\n from changedetectionio.processors import available_processors\n available = [p[0] for p in available_processors()]\n if extras['processor'] not in available:\n return f\"Invalid processor '{extras['processor']}'. Available processors: {', '.join(available)}\", 400\n\n # Validate fetch_backend if provided\n if 'fetch_backend' in extras:\n from changedetectionio.content_fetchers import available_fetchers\n available = [f[0] for f in available_fetchers()]\n # Also allow 'system' and extra_browser_* patterns\n is_valid = (\n extras['fetch_backend'] == 'system' or\n extras['fetch_backend'] in available or\n extras['fetch_backend'].startswith('extra_browser_')\n )\n if not is_valid:\n return f\"Invalid fetch_backend '{extras['fetch_backend']}'. Available: system, {', '.join(available)}\", 400\n\n # Validate notification_urls if provided\n if 'notification_urls' in extras:\n from wtforms import ValidationError\n from changedetectionio.api.Notifications import validate_notification_urls\n try:\n validate_notification_urls(extras['notification_urls'])\n except ValidationError as e:\n return f\"Invalid notification_urls: {str(e)}\", 400\n\n urls = request.get_data().decode('utf8').splitlines()\n # Clean and validate URLs upfront\n urls_to_import = []\n for url in urls:\n url = url.strip()\n if not len(url):\n continue\n\n # Validate URL\n if not is_safe_valid_url(url):\n return f\"Invalid or unsupported URL - {url}\", 400\n\n # Check for duplicates if dedupe is enabled\n if dedupe and self.datastore.url_exists(url):\n continue\n\n urls_to_import.append(url)\n\n # For small imports, process synchronously for immediate feedback\n if len(urls_to_import) < IMPORT_SWITCH_TO_BACKGROUND_THRESHOLD:\n added = []\n for url in urls_to_import:\n new_uuid = self.datastore.add_watch(url=url, extras=extras, tag=tags, tag_uuids=tag_uuids)\n added.append(new_uuid)\n return added, 200\n\n # For large imports (>= 20), process in background thread\n else:\n import threading\n from loguru import logger\n\n def import_watches_background():\n \"\"\"Background thread to import watches - discarded after completion.\"\"\"\n try:\n added_count = 0\n for url in urls_to_import:\n try:\n self.datastore.add_watch(url=url, extras=extras, tag=tags, tag_uuids=tag_uuids)\n added_count += 1\n except Exception as e:\n logger.error(f\"Error importing URL {url}: {e}\")\n\n logger.info(f\"Background import complete: {added_count} watches created\")\n except Exception as e:\n logger.error(f\"Error in background import: {e}\")\n\n # Start background thread and return immediately\n thread = threading.Thread(target=import_watches_background, daemon=True, name=\"ImportWatches-Background\")\n thread.start()\n\n return {'status': f'Importing {len(urls_to_import)} URLs in background', 'count': len(urls_to_import)}, 202" }, { "path": "changedetectionio/api/Notifications.py", "content": "from flask_restful import Resource, abort\nfrom flask import request\nfrom . import auth, validate_openapi_request\n\nclass Notifications(Resource):\n def __init__(self, **kwargs):\n # datastore is a black box dependency\n self.datastore = kwargs['datastore']\n\n @auth.check_token\n @validate_openapi_request('getNotifications')\n def get(self):\n \"\"\"Return Notification URL List.\"\"\"\n\n notification_urls = self.datastore.data.get('settings', {}).get('application', {}).get('notification_urls', []) \n\n return {\n 'notification_urls': notification_urls,\n }, 200\n \n @auth.check_token\n @validate_openapi_request('addNotifications')\n def post(self):\n \"\"\"Create Notification URLs.\"\"\"\n\n json_data = request.get_json()\n notification_urls = json_data.get(\"notification_urls\", [])\n\n from wtforms import ValidationError\n try:\n validate_notification_urls(notification_urls)\n except ValidationError as e:\n return str(e), 400\n\n added_urls = []\n\n for url in notification_urls:\n clean_url = url.strip()\n added_url = self.datastore.add_notification_url(clean_url)\n if added_url:\n added_urls.append(added_url)\n\n if not added_urls:\n return \"No valid notification URLs were added\", 400\n\n return {'notification_urls': added_urls}, 201\n \n @auth.check_token\n @validate_openapi_request('replaceNotifications')\n def put(self):\n \"\"\"Replace Notification URLs.\"\"\"\n json_data = request.get_json()\n notification_urls = json_data.get(\"notification_urls\", [])\n\n from wtforms import ValidationError\n try:\n validate_notification_urls(notification_urls)\n except ValidationError as e:\n return str(e), 400\n \n if not isinstance(notification_urls, list):\n return \"Invalid input format\", 400\n\n clean_urls = [url.strip() for url in notification_urls if isinstance(url, str)]\n self.datastore.data['settings']['application']['notification_urls'] = clean_urls\n self.datastore.commit()\n\n return {'notification_urls': clean_urls}, 200\n \n @auth.check_token\n @validate_openapi_request('deleteNotifications')\n def delete(self):\n \"\"\"Delete Notification URLs.\"\"\"\n\n json_data = request.get_json()\n urls_to_delete = json_data.get(\"notification_urls\", [])\n if not isinstance(urls_to_delete, list):\n abort(400, message=\"Expected a list of notification URLs.\")\n\n notification_urls = self.datastore.data['settings']['application'].get('notification_urls', [])\n deleted = []\n\n for url in urls_to_delete:\n clean_url = url.strip()\n if clean_url in notification_urls:\n notification_urls.remove(clean_url)\n deleted.append(clean_url)\n\n if not deleted:\n abort(400, message=\"No matching notification URLs found.\")\n\n self.datastore.data['settings']['application']['notification_urls'] = notification_urls\n self.datastore.commit()\n\n return 'OK', 204\n \ndef validate_notification_urls(notification_urls):\n from changedetectionio.forms import ValidateAppRiseServers\n validator = ValidateAppRiseServers()\n class DummyForm: pass\n dummy_form = DummyForm()\n field = type(\"Field\", (object,), {\"data\": notification_urls, \"gettext\": lambda self, x: x})()\n validator(dummy_form, field)" }, { "path": "changedetectionio/api/Search.py", "content": "from flask_restful import Resource, abort\nfrom flask import request\nfrom . import auth, validate_openapi_request\n\nclass Search(Resource):\n def __init__(self, **kwargs):\n # datastore is a black box dependency\n self.datastore = kwargs['datastore']\n\n @auth.check_token\n @validate_openapi_request('searchWatches')\n def get(self):\n \"\"\"Search for watches by URL or title text.\"\"\"\n query = request.args.get('q', '').strip()\n tag_limit = request.args.get('tag', '').strip()\n from changedetectionio.strtobool import strtobool\n partial = bool(strtobool(request.args.get('partial', '0'))) if 'partial' in request.args else False\n\n # Require a search query\n if not query:\n abort(400, message=\"Search query 'q' parameter is required\")\n\n # Use the search function from the datastore\n matching_uuids = self.datastore.search_watches_for_url(query=query, tag_limit=tag_limit, partial=partial)\n\n # Build the response with watch details\n results = {}\n for uuid in matching_uuids:\n watch = self.datastore.data['watching'].get(uuid)\n results[uuid] = {\n 'last_changed': watch.last_changed,\n 'last_checked': watch['last_checked'],\n 'last_error': watch['last_error'],\n 'title': watch['title'],\n 'url': watch['url'],\n 'viewed': watch.viewed\n }\n\n return results, 200" }, { "path": "changedetectionio/api/Spec.py", "content": "import functools\nfrom flask import make_response\nfrom flask_restful import Resource\n\n\n@functools.cache\ndef _get_spec_yaml():\n \"\"\"Build and cache the merged spec as a YAML string (only serialized once per process).\"\"\"\n import yaml\n from changedetectionio.api import build_merged_spec_dict\n return yaml.dump(build_merged_spec_dict(), default_flow_style=False, allow_unicode=True)\n\n\nclass Spec(Resource):\n def get(self):\n \"\"\"Return the merged OpenAPI spec including all registered processor extensions.\"\"\"\n return make_response(\n _get_spec_yaml(),\n 200,\n {'Content-Type': 'application/yaml'}\n )\n" }, { "path": "changedetectionio/api/SystemInfo.py", "content": "from flask_restful import Resource\nfrom . import auth, validate_openapi_request\n\n\nclass SystemInfo(Resource):\n def __init__(self, **kwargs):\n # datastore is a black box dependency\n self.datastore = kwargs['datastore']\n self.update_q = kwargs['update_q']\n\n @auth.check_token\n @validate_openapi_request('getSystemInfo')\n def get(self):\n \"\"\"Return system info.\"\"\"\n import time\n overdue_watches = []\n\n # Check all watches and report which have not been checked but should have been\n\n for uuid, watch in self.datastore.data.get('watching', {}).items():\n # see if now - last_checked is greater than the time that should have been\n # this is not super accurate (maybe they just edited it) but better than nothing\n t = watch.threshold_seconds()\n if not t:\n # Use the system wide default\n t = self.datastore.threshold_seconds\n\n time_since_check = time.time() - watch.get('last_checked')\n\n # Allow 5 minutes of grace time before we decide it's overdue\n if time_since_check - (5 * 60) > t:\n overdue_watches.append(uuid)\n from changedetectionio import __version__ as main_version\n return {\n 'queue_size': self.update_q.qsize(),\n 'overdue_watches': overdue_watches,\n 'uptime': round(time.time() - self.datastore.start_time, 2),\n 'watch_count': len(self.datastore.data.get('watching', {})),\n 'version': main_version\n }, 200" }, { "path": "changedetectionio/api/Tags.py", "content": "from changedetectionio import queuedWatchMetaData\nfrom changedetectionio import worker_pool\nfrom flask_restful import abort, Resource\nfrom loguru import logger\n\nimport threading\nfrom flask import request\nfrom . import auth\n\nfrom . import validate_openapi_request\n\n\nclass Tag(Resource):\n def __init__(self, **kwargs):\n # datastore is a black box dependency\n self.datastore = kwargs['datastore']\n self.update_q = kwargs['update_q']\n\n # Get information about a single tag\n # curl http://localhost:5000/api/v1/tag/\n @auth.check_token\n @validate_openapi_request('getTag')\n def get(self, uuid):\n \"\"\"Get data for a single tag/group, toggle notification muting, or recheck all.\"\"\"\n tag = self.datastore.data['settings']['application']['tags'].get(uuid)\n if not tag:\n abort(404, message=f'No tag exists with the UUID of {uuid}')\n\n if request.args.get('recheck'):\n # Recheck all watches with this tag, including muted\n # First collect watches to queue\n watches_to_queue = []\n for k in sorted(self.datastore.data['watching'].items(), key=lambda item: item[1].get('last_checked', 0)):\n watch_uuid = k[0]\n watch = k[1]\n if not watch['paused'] and tag['uuid'] in watch['tags']:\n watches_to_queue.append(watch_uuid)\n\n # If less than 20 watches, queue synchronously for immediate feedback\n if len(watches_to_queue) < 20:\n for watch_uuid in watches_to_queue:\n worker_pool.queue_item_async_safe(self.update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid}))\n return {'status': f'OK, queued {len(watches_to_queue)} watches for rechecking'}, 200\n else:\n # 20+ watches - queue in background thread to avoid blocking API response\n def queue_watches_background():\n \"\"\"Background thread to queue watches - discarded after completion.\"\"\"\n try:\n for watch_uuid in watches_to_queue:\n worker_pool.queue_item_async_safe(self.update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid}))\n logger.info(f\"Background queueing complete for tag {tag['uuid']}: {len(watches_to_queue)} watches queued\")\n except Exception as e:\n logger.error(f\"Error in background queueing for tag {tag['uuid']}: {e}\")\n\n # Start background thread and return immediately\n thread = threading.Thread(target=queue_watches_background, daemon=True, name=f\"QueueTag-{tag['uuid'][:8]}\")\n thread.start()\n\n return {'status': f'OK, queueing {len(watches_to_queue)} watches in background'}, 202\n\n if request.args.get('muted', '') == 'muted':\n tag['notification_muted'] = True\n tag.commit()\n return \"OK\", 200\n elif request.args.get('muted', '') == 'unmuted':\n tag['notification_muted'] = False\n tag.commit()\n return \"OK\", 200\n\n # Filter out Watch-specific runtime fields that don't apply to Tags (yet)\n # TODO: Future enhancement - aggregate these values from all Watches that have this tag:\n # - check_count: sum of all watches' check_count\n # - last_checked: most recent last_checked from all watches\n # - last_changed: most recent last_changed from all watches\n # - consecutive_filter_failures: count of watches with failures\n # - etc.\n # These come from watch_base inheritance but currently have no meaningful value for Tags\n watch_only_fields = {\n 'browser_steps_last_error_step', 'check_count', 'consecutive_filter_failures',\n 'content-type', 'fetch_time', 'last_changed', 'last_checked', 'last_error',\n 'last_notification_error', 'last_viewed', 'notification_alert_count',\n 'page_title', 'previous_md5', 'remote_server_reply'\n }\n\n # Create clean tag dict without Watch-specific fields\n clean_tag = {k: v for k, v in tag.items() if k not in watch_only_fields}\n\n return clean_tag\n\n @auth.check_token\n @validate_openapi_request('deleteTag')\n def delete(self, uuid):\n \"\"\"Delete a tag/group and remove it from all watches.\"\"\"\n if not self.datastore.data['settings']['application']['tags'].get(uuid):\n abort(400, message='No tag exists with the UUID of {}'.format(uuid))\n\n # Delete the tag, and any tag reference\n del self.datastore.data['settings']['application']['tags'][uuid]\n\n # Remove tag from all watches\n for watch_uuid, watch in self.datastore.data['watching'].items():\n if watch.get('tags') and uuid in watch['tags']:\n watch['tags'].remove(uuid)\n watch.commit()\n\n return 'OK', 204\n\n @auth.check_token\n @validate_openapi_request('updateTag')\n def put(self, uuid):\n \"\"\"Update tag information.\"\"\"\n tag = self.datastore.data['settings']['application']['tags'].get(uuid)\n if not tag:\n abort(404, message='No tag exists with the UUID of {}'.format(uuid))\n\n # Make a mutable copy of request.json for modification\n json_data = dict(request.json)\n\n # Validate notification_urls if provided\n if 'notification_urls' in json_data:\n from wtforms import ValidationError\n from changedetectionio.api.Notifications import validate_notification_urls\n try:\n notification_urls = json_data.get('notification_urls', [])\n validate_notification_urls(notification_urls)\n except ValidationError as e:\n return str(e), 400\n\n # Filter out readOnly fields (extracted from OpenAPI spec Tag schema)\n # These are system-managed fields that should never be user-settable\n from . import get_readonly_tag_fields\n readonly_fields = get_readonly_tag_fields()\n\n # Tag model inherits from watch_base but has no @property attributes of its own\n # So we only need to filter readOnly fields\n for field in readonly_fields:\n json_data.pop(field, None)\n\n # Validate remaining fields - reject truly unknown fields\n # Get valid fields from Tag schema\n from . import get_tag_schema_properties\n valid_fields = set(get_tag_schema_properties().keys())\n\n # Check for unknown fields\n unknown_fields = set(json_data.keys()) - valid_fields\n if unknown_fields:\n return f\"Unknown field(s): {', '.join(sorted(unknown_fields))}\", 400\n\n tag.update(json_data)\n tag.commit()\n\n # Clear checksums for all watches using this tag to force reprocessing\n # Tag changes affect inherited configuration\n cleared_count = self.datastore.clear_checksums_for_tag(uuid)\n logger.info(f\"Tag {uuid} updated via API, cleared {cleared_count} watch checksums\")\n\n return \"OK\", 200\n\n\n @auth.check_token\n @validate_openapi_request('createTag')\n def post(self):\n \"\"\"Create a single tag/group.\"\"\"\n\n json_data = request.get_json()\n title = json_data.get(\"title\",'').strip()\n\n # Validate that only valid fields are provided\n # Get valid fields from Tag schema\n from . import get_tag_schema_properties\n valid_fields = set(get_tag_schema_properties().keys())\n\n # Check for unknown fields\n unknown_fields = set(json_data.keys()) - valid_fields\n if unknown_fields:\n return f\"Unknown field(s): {', '.join(sorted(unknown_fields))}\", 400\n\n new_uuid = self.datastore.add_tag(title=title)\n if new_uuid:\n # Apply any extra fields (e.g. processor_config_restock_diff) beyond just title\n extra = {k: v for k, v in json_data.items() if k != 'title'}\n if extra:\n tag = self.datastore.data['settings']['application']['tags'].get(new_uuid)\n if tag:\n tag.update(extra)\n tag.commit()\n return {'uuid': new_uuid}, 201\n else:\n return \"Invalid or unsupported tag\", 400\n\nclass Tags(Resource):\n def __init__(self, **kwargs):\n # datastore is a black box dependency\n self.datastore = kwargs['datastore']\n\n @auth.check_token\n @validate_openapi_request('listTags')\n def get(self):\n \"\"\"List tags/groups.\"\"\"\n result = {}\n for uuid, tag in self.datastore.data['settings']['application']['tags'].items():\n result[uuid] = {\n 'date_created': tag.get('date_created', 0),\n 'notification_muted': tag.get('notification_muted', False),\n 'title': tag.get('title', ''),\n 'uuid': tag.get('uuid')\n }\n\n return result, 200" }, { "path": "changedetectionio/api/Watch.py", "content": "import os\nimport threading\n\nfrom changedetectionio.validate_url import is_safe_valid_url\nfrom changedetectionio.favicon_utils import get_favicon_mime_type\n\nfrom . import auth\nfrom changedetectionio import queuedWatchMetaData, strtobool\nfrom changedetectionio import worker_pool\nfrom flask import request, make_response, send_from_directory\nfrom flask_restful import abort, Resource\nfrom loguru import logger\nimport copy\n\nfrom . import validate_openapi_request, get_readonly_watch_fields\nfrom ..notification import valid_notification_formats\nfrom ..notification.handler import newline_re\n\n\ndef validate_time_between_check_required(json_data):\n \"\"\"\n Validate that at least one time interval is specified when not using default settings.\n Returns None if valid, or error message string if invalid.\n Defaults to using global settings if time_between_check_use_default is not provided.\n \"\"\"\n # Default to using global settings if not specified\n use_default = json_data.get('time_between_check_use_default', True)\n\n # If using default settings, no validation needed\n if use_default:\n return None\n\n # If not using defaults, check if time_between_check exists and has at least one non-zero value\n time_check = json_data.get('time_between_check')\n if not time_check:\n # No time_between_check provided and not using defaults - this is an error\n return \"At least one time interval (weeks, days, hours, minutes, or seconds) must be specified when not using global settings.\"\n\n # time_between_check exists, check if it has at least one non-zero value\n if any([\n (time_check.get('weeks') or 0) > 0,\n (time_check.get('days') or 0) > 0,\n (time_check.get('hours') or 0) > 0,\n (time_check.get('minutes') or 0) > 0,\n (time_check.get('seconds') or 0) > 0\n ]):\n return None\n\n # time_between_check exists but all values are 0 or empty - this is an error\n return \"At least one time interval (weeks, days, hours, minutes, or seconds) must be specified when not using global settings.\"\n\n\nclass Watch(Resource):\n def __init__(self, **kwargs):\n # datastore is a black box dependency\n self.datastore = kwargs['datastore']\n self.update_q = kwargs['update_q']\n\n # Get information about a single watch, excluding the history list (can be large)\n # curl http://localhost:5000/api/v1/watch/\n # @todo - version2 - ?muted and ?paused should be able to be called together, return the watch struct not \"OK\"\n # ?recheck=true\n @auth.check_token\n @validate_openapi_request('getWatch')\n def get(self, uuid):\n \"\"\"Get information about a single watch, recheck, pause, or mute.\"\"\"\n # Get watch reference first (for pause/mute operations)\n watch_obj = self.datastore.data['watching'].get(uuid)\n if not watch_obj:\n abort(404, message='No watch exists with the UUID of {}'.format(uuid))\n\n # Create a dict copy for JSON response (with lock for thread safety)\n # This is much faster than deepcopy and doesn't copy the datastore reference\n # WARNING: dict() is a SHALLOW copy - nested dicts are shared with original!\n # Only safe because we only ADD scalar properties (line 97-101), never modify nested dicts\n # If you need to modify nested dicts, use: from copy import deepcopy; watch = deepcopy(dict(watch_obj))\n with self.datastore.lock:\n watch = dict(watch_obj)\n\n if request.args.get('recheck'):\n worker_pool.queue_item_async_safe(self.update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))\n return \"OK\", 200\n if request.args.get('paused', '') == 'paused':\n watch_obj.pause()\n watch_obj.commit()\n return \"OK\", 200\n elif request.args.get('paused', '') == 'unpaused':\n watch_obj.unpause()\n watch_obj.commit()\n return \"OK\", 200\n if request.args.get('muted', '') == 'muted':\n watch_obj.mute()\n watch_obj.commit()\n return \"OK\", 200\n elif request.args.get('muted', '') == 'unmuted':\n watch_obj.unmute()\n watch_obj.commit()\n return \"OK\", 200\n\n # Return without history, get that via another API call\n # Properties are not returned as a JSON, so add the required props manually\n watch['history_n'] = watch_obj.history_n\n # attr .last_changed will check for the last written text snapshot on change\n watch['last_changed'] = watch_obj.last_changed\n watch['viewed'] = watch_obj.viewed\n watch['link'] = watch_obj.link,\n\n return watch\n\n @auth.check_token\n @validate_openapi_request('deleteWatch')\n def delete(self, uuid):\n \"\"\"Delete a watch and related history.\"\"\"\n if not self.datastore.data['watching'].get(uuid):\n abort(400, message='No watch exists with the UUID of {}'.format(uuid))\n\n self.datastore.delete(uuid)\n return 'OK', 204\n\n @auth.check_token\n @validate_openapi_request('updateWatch')\n def put(self, uuid):\n \"\"\"Update watch information.\"\"\"\n watch = self.datastore.data['watching'].get(uuid)\n if not watch:\n abort(404, message='No watch exists with the UUID of {}'.format(uuid))\n\n if request.json.get('proxy'):\n plist = self.datastore.proxy_list\n if not plist or request.json.get('proxy') not in plist:\n proxy_list_str = ', '.join(plist) if plist else 'none configured'\n return f\"Invalid proxy choice, currently supported proxies are '{proxy_list_str}'\", 400\n\n # Validate time_between_check when not using defaults\n validation_error = validate_time_between_check_required(request.json)\n if validation_error:\n return validation_error, 400\n\n # Validate notification_urls if provided\n if 'notification_urls' in request.json:\n from wtforms import ValidationError\n from changedetectionio.api.Notifications import validate_notification_urls\n try:\n notification_urls = request.json.get('notification_urls', [])\n validate_notification_urls(notification_urls)\n except ValidationError as e:\n return str(e), 400\n\n # XSS etc protection - validate URL if it's being updated\n if 'url' in request.json:\n new_url = request.json.get('url')\n\n # URL must be a non-empty string\n if new_url is None:\n return \"URL cannot be null\", 400\n\n if not isinstance(new_url, str):\n return \"URL must be a string\", 400\n\n if not new_url.strip():\n return \"URL cannot be empty or whitespace only\", 400\n\n if not is_safe_valid_url(new_url.strip()):\n return \"Invalid or unsupported URL format. URL must use http://, https://, or ftp:// protocol\", 400\n\n # Handle processor-config-* fields separately (save to JSON, not datastore)\n from changedetectionio import processors\n\n # Make a mutable copy of request.json for modification\n json_data = dict(request.json)\n\n # Extract and remove processor config fields from json_data\n processor_config_data = processors.extract_processor_config_from_form_data(json_data)\n\n # Filter out readOnly fields (extracted from OpenAPI spec Watch schema)\n # These are system-managed fields that should never be user-settable\n readonly_fields = get_readonly_watch_fields()\n\n # Also filter out @property attributes (computed/derived values from the model)\n # These are not stored and should be ignored in PUT requests\n from changedetectionio.model.Watch import model as WatchModel\n property_fields = WatchModel.get_property_names()\n\n # Combine both sets of fields to ignore\n fields_to_ignore = readonly_fields | property_fields\n\n # Remove all ignored fields from update data\n for field in fields_to_ignore:\n json_data.pop(field, None)\n\n # Validate remaining fields - reject truly unknown fields\n # Get valid fields from WatchBase schema\n from . import get_watch_schema_properties\n valid_fields = set(get_watch_schema_properties().keys())\n\n # Also allow last_viewed (explicitly defined in UpdateWatch schema)\n valid_fields.add('last_viewed')\n\n # Check for unknown fields\n unknown_fields = set(json_data.keys()) - valid_fields\n if unknown_fields:\n return f\"Unknown field(s): {', '.join(sorted(unknown_fields))}\", 400\n\n # Update watch with regular (non-processor-config) fields\n watch.update(json_data)\n watch.commit()\n\n # Save processor config to JSON file\n processors.save_processor_config(self.datastore, uuid, processor_config_data)\n\n return \"OK\", 200\n\n\nclass WatchHistory(Resource):\n def __init__(self, **kwargs):\n # datastore is a black box dependency\n self.datastore = kwargs['datastore']\n\n # Get a list of available history for a watch by UUID\n # curl http://localhost:5000/api/v1/watch//history\n @auth.check_token\n @validate_openapi_request('getWatchHistory')\n def get(self, uuid):\n \"\"\"Get a list of all historical snapshots available for a watch.\"\"\"\n watch = self.datastore.data['watching'].get(uuid)\n if not watch:\n abort(404, message='No watch exists with the UUID of {}'.format(uuid))\n return watch.history, 200\n\n\nclass WatchSingleHistory(Resource):\n def __init__(self, **kwargs):\n # datastore is a black box dependency\n self.datastore = kwargs['datastore']\n\n @auth.check_token\n @validate_openapi_request('getWatchSnapshot')\n def get(self, uuid, timestamp):\n \"\"\"Get single snapshot from watch.\"\"\"\n watch = self.datastore.data['watching'].get(uuid)\n if not watch:\n abort(404, message=f\"No watch exists with the UUID of {uuid}\")\n\n if not len(watch.history):\n abort(404, message=f\"Watch found but no history exists for the UUID {uuid}\")\n\n if timestamp == 'latest':\n timestamp = list(watch.history.keys())[-1]\n\n # Validate that the timestamp exists in history\n if timestamp not in watch.history:\n abort(404, message=f\"No history snapshot found for timestamp '{timestamp}'\")\n\n if request.args.get('html'):\n content = watch.get_fetched_html(timestamp)\n if content:\n response = make_response(content, 200)\n response.mimetype = \"text/html\"\n else:\n response = make_response(\"No content found\", 404)\n response.mimetype = \"text/plain\"\n else:\n content = watch.get_history_snapshot(timestamp=timestamp)\n response = make_response(content, 200)\n response.mimetype = \"text/plain\"\n\n return response\n\nclass WatchHistoryDiff(Resource):\n \"\"\"\n Generate diff between two historical snapshots.\n\n Note: This API endpoint currently returns text-based diffs and works best\n with the text_json_diff processor. Future processor types (like image_diff,\n restock_diff) may want to implement their own specialized API endpoints\n for returning processor-specific data (e.g., price charts, image comparisons).\n\n The web UI diff page (/diff/) is processor-aware and delegates rendering\n to processors/{type}/difference.py::render() for processor-specific visualizations.\n \"\"\"\n def __init__(self, **kwargs):\n # datastore is a black box dependency\n self.datastore = kwargs['datastore']\n\n @auth.check_token\n @validate_openapi_request('getWatchHistoryDiff')\n def get(self, uuid, from_timestamp, to_timestamp):\n \"\"\"Generate diff between two historical snapshots.\"\"\"\n from changedetectionio import diff\n from changedetectionio.notification.handler import apply_service_tweaks\n\n watch = self.datastore.data['watching'].get(uuid)\n if not watch:\n abort(404, message=f\"No watch exists with the UUID of {uuid}\")\n\n if not len(watch.history):\n abort(404, message=f\"Watch found but no history exists for the UUID {uuid}\")\n\n history_keys = list(watch.history.keys())\n\n # Handle 'latest' keyword for to_timestamp\n if to_timestamp == 'latest':\n to_timestamp = history_keys[-1]\n\n # Handle 'previous' keyword for from_timestamp (second-most-recent)\n if from_timestamp == 'previous':\n if len(history_keys) < 2:\n abort(404, message=f\"Not enough history entries. Need at least 2 snapshots for 'previous'\")\n from_timestamp = history_keys[-2]\n\n # Validate timestamps exist\n if from_timestamp not in watch.history:\n abort(404, message=f\"From timestamp {from_timestamp} not found in watch history\")\n if to_timestamp not in watch.history:\n abort(404, message=f\"To timestamp {to_timestamp} not found in watch history\")\n\n # Get the format parameter (default to 'text')\n output_format = request.args.get('format', 'text').lower()\n\n # Validate format\n if output_format not in valid_notification_formats.keys():\n abort(400, message=f\"Invalid format. Must be one of: {', '.join(valid_notification_formats.keys())}\")\n\n # Get the word_diff parameter (default to False - line-level mode)\n word_diff = strtobool(request.args.get('word_diff', 'false'))\n\n # Get the no_markup parameter (default to False)\n no_markup = strtobool(request.args.get('no_markup', 'false'))\n\n # Retrieve snapshot contents\n from_version_file_contents = watch.get_history_snapshot(from_timestamp)\n to_version_file_contents = watch.get_history_snapshot(to_timestamp)\n\n # Get diff preferences from query parameters (matching UI preferences in DIFF_PREFERENCES_CONFIG)\n # Support both 'type' (UI parameter) and 'word_diff' (API parameter) for backward compatibility\n diff_type = request.args.get('type', 'diffLines')\n if diff_type == 'diffWords':\n word_diff = True\n\n # Get boolean diff preferences with defaults from DIFF_PREFERENCES_CONFIG\n changes_only = strtobool(request.args.get('changesOnly', 'false'))\n ignore_whitespace = strtobool(request.args.get('ignoreWhitespace', 'false'))\n include_removed = strtobool(request.args.get('removed', 'true'))\n include_added = strtobool(request.args.get('added', 'true'))\n include_replaced = strtobool(request.args.get('replaced', 'true'))\n\n # Generate the diff with all preferences\n content = diff.render_diff(\n previous_version_file_contents=from_version_file_contents,\n newest_version_file_contents=to_version_file_contents,\n ignore_junk=ignore_whitespace,\n include_equal=not changes_only,\n include_removed=include_removed,\n include_added=include_added,\n include_replaced=include_replaced,\n word_diff=word_diff,\n )\n\n # Skip formatting if no_markup is set\n if no_markup:\n mimetype = \"text/plain\"\n else:\n # Apply formatting based on the requested format\n if output_format == 'htmlcolor':\n from changedetectionio.notification.handler import apply_html_color_to_body\n content = apply_html_color_to_body(n_body=content)\n mimetype = \"text/html\"\n else:\n # Apply service tweaks for text/html formats\n # Pass empty URL and title as they're not used for the placeholder replacement we need\n _, content, _ = apply_service_tweaks(\n url='',\n n_body=content,\n n_title='',\n requested_output_format=output_format\n )\n mimetype = \"text/html\" if output_format == 'html' else \"text/plain\"\n\n if 'html' in output_format:\n content = newline_re.sub('
\\r\\n', content)\n\n response = make_response(content, 200)\n response.mimetype = mimetype\n return response\n\n\nclass WatchFavicon(Resource):\n def __init__(self, **kwargs):\n # datastore is a black box dependency\n self.datastore = kwargs['datastore']\n\n @auth.check_token\n @validate_openapi_request('getWatchFavicon')\n def get(self, uuid):\n \"\"\"Get favicon for a watch.\"\"\"\n watch = self.datastore.data['watching'].get(uuid)\n if not watch:\n abort(404, message=f\"No watch exists with the UUID of {uuid}\")\n\n favicon_filename = watch.get_favicon_filename()\n if favicon_filename:\n # Use cached MIME type detection\n filepath = os.path.join(watch.data_dir, favicon_filename)\n mime = get_favicon_mime_type(filepath)\n\n response = make_response(send_from_directory(watch.data_dir, favicon_filename))\n response.headers['Content-type'] = mime\n response.headers['Cache-Control'] = 'max-age=300, must-revalidate' # Cache for 5 minutes, then revalidate\n return response\n\n abort(404, message=f'No Favicon available for {uuid}')\n\n\nclass CreateWatch(Resource):\n def __init__(self, **kwargs):\n # datastore is a black box dependency\n self.datastore = kwargs['datastore']\n self.update_q = kwargs['update_q']\n\n @auth.check_token\n @validate_openapi_request('createWatch')\n def post(self):\n \"\"\"Create a single watch.\"\"\"\n\n json_data = request.get_json()\n url = json_data['url'].strip()\n\n if not is_safe_valid_url(url):\n return \"Invalid or unsupported URL\", 400\n\n if json_data.get('proxy'):\n plist = self.datastore.proxy_list\n if not plist or json_data.get('proxy') not in plist:\n proxy_list_str = ', '.join(plist) if plist else 'none configured'\n return f\"Invalid proxy choice, currently supported proxies are '{proxy_list_str}'\", 400\n\n # Validate time_between_check when not using defaults\n validation_error = validate_time_between_check_required(json_data)\n if validation_error:\n return validation_error, 400\n\n # Validate notification_urls if provided\n if 'notification_urls' in json_data:\n from wtforms import ValidationError\n from changedetectionio.api.Notifications import validate_notification_urls\n try:\n notification_urls = json_data.get('notification_urls', [])\n validate_notification_urls(notification_urls)\n except ValidationError as e:\n return str(e), 400\n\n # Handle processor-config-* fields separately (save to JSON, not watch)\n from changedetectionio import processors\n\n extras = copy.deepcopy(json_data)\n\n # Extract and remove processor config fields from extras\n processor_config_data = processors.extract_processor_config_from_form_data(extras)\n\n # Because we renamed 'tag' to 'tags' but don't want to change the API (can do this in v2 of the API)\n tags = None\n if extras.get('tag'):\n tags = extras.get('tag')\n del extras['tag']\n\n del extras['url']\n\n new_uuid = self.datastore.add_watch(url=url, extras=extras, tag=tags)\n\n # Save processor config to separate JSON file\n if new_uuid and processor_config_data:\n processors.save_processor_config(self.datastore, new_uuid, processor_config_data)\n if new_uuid:\n# Dont queue because the scheduler will check that it hasnt been checked before anyway\n# worker_pool.queue_item_async_safe(self.update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': new_uuid}))\n return {'uuid': new_uuid}, 201\n else:\n # Check if it was a limit issue\n page_watch_limit = os.getenv('PAGE_WATCH_LIMIT')\n if page_watch_limit:\n try:\n page_watch_limit = int(page_watch_limit)\n current_watch_count = len(self.datastore.data['watching'])\n if current_watch_count >= page_watch_limit:\n return f\"Watch limit reached ({current_watch_count}/{page_watch_limit} watches). Cannot add more watches.\", 429\n except ValueError:\n pass\n return \"Invalid or unsupported URL\", 400\n\n @auth.check_token\n @validate_openapi_request('listWatches')\n def get(self):\n \"\"\"List watches.\"\"\"\n list = {}\n\n tag_limit = request.args.get('tag', '').lower()\n for uuid, watch in self.datastore.data['watching'].items():\n # Watch tags by name (replace the other calls?)\n tags = self.datastore.get_all_tags_for_watch(uuid=uuid)\n if tag_limit and not any(v.get('title').lower() == tag_limit for k, v in tags.items()):\n continue\n\n list[uuid] = {\n 'last_changed': watch.last_changed,\n 'last_checked': watch['last_checked'],\n 'last_error': watch['last_error'],\n 'link': watch.link,\n 'page_title': watch['page_title'],\n 'tags': [*tags], # Unpack dict keys to list (can't use list() since variable named 'list')\n 'title': watch['title'],\n 'url': watch['url'],\n 'viewed': watch.viewed\n }\n\n if request.args.get('recheck_all'):\n # Collect all watches to queue\n watches_to_queue = self.datastore.data['watching'].keys()\n\n # If less than 20 watches, queue synchronously for immediate feedback\n if len(watches_to_queue) < 20:\n # Get already queued/running UUIDs once (efficient)\n queued_uuids = set(self.update_q.get_queued_uuids())\n running_uuids = set(worker_pool.get_running_uuids())\n\n # Filter out watches that are already queued or running\n watches_to_queue_filtered = [\n uuid for uuid in watches_to_queue\n if uuid not in queued_uuids and uuid not in running_uuids\n ]\n\n # Queue only the filtered watches\n for uuid in watches_to_queue_filtered:\n worker_pool.queue_item_async_safe(self.update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))\n\n # Provide feedback about skipped watches\n skipped_count = len(watches_to_queue) - len(watches_to_queue_filtered)\n if skipped_count > 0:\n return {'status': f'OK, queued {len(watches_to_queue_filtered)} watches for rechecking ({skipped_count} already queued or running)'}, 200\n else:\n return {'status': f'OK, queued {len(watches_to_queue_filtered)} watches for rechecking'}, 200\n else:\n # 20+ watches - queue in background thread to avoid blocking API response\n # Capture queued/running state before background thread\n queued_uuids = set(self.update_q.get_queued_uuids())\n running_uuids = set(worker_pool.get_running_uuids())\n\n def queue_all_watches_background():\n \"\"\"Background thread to queue all watches - discarded after completion.\"\"\"\n try:\n queued_count = 0\n skipped_count = 0\n for uuid in watches_to_queue:\n # Check if already queued or running (state captured at start)\n if uuid not in queued_uuids and uuid not in running_uuids:\n worker_pool.queue_item_async_safe(self.update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))\n queued_count += 1\n else:\n skipped_count += 1\n\n logger.info(f\"Background queueing complete: {queued_count} watches queued, {skipped_count} skipped (already queued/running)\")\n except Exception as e:\n logger.error(f\"Error in background queueing all watches: {e}\")\n\n # Start background thread and return immediately\n thread = threading.Thread(target=queue_all_watches_background, daemon=True, name=\"QueueAllWatches-Background\")\n thread.start()\n\n return {'status': f'OK, queueing {len(watches_to_queue)} watches in background'}, 202\n\n return list, 200\n" }, { "path": "changedetectionio/api/__init__.py", "content": "import functools\nfrom flask import request, abort\nfrom loguru import logger\n\n@functools.cache\ndef build_merged_spec_dict():\n \"\"\"\n Load the base OpenAPI spec and merge in any per-processor api.yaml extensions.\n\n Each processor can provide an api.yaml file alongside its __init__.py that defines\n additional schemas (e.g., processor_config_restock_diff). These are merged into\n WatchBase.properties so the spec accurately reflects what the API accepts.\n\n Plugin processors (via pluggy) are also supported - they just need an api.yaml\n next to their processor module.\n\n Returns the merged dict (cached - do not mutate the returned value).\n \"\"\"\n import os\n import yaml\n\n spec_path = os.path.join(os.path.dirname(__file__), '../../docs/api-spec.yaml')\n if not os.path.exists(spec_path):\n spec_path = os.path.join(os.path.dirname(__file__), '../docs/api-spec.yaml')\n\n with open(spec_path, 'r', encoding='utf-8') as f:\n spec_dict = yaml.safe_load(f)\n\n try:\n from changedetectionio.processors import find_processors, get_parent_module\n for module, proc_name in find_processors():\n parent = get_parent_module(module)\n if not parent or not hasattr(parent, '__file__'):\n continue\n api_yaml_path = os.path.join(os.path.dirname(parent.__file__), 'api.yaml')\n if not os.path.exists(api_yaml_path):\n continue\n with open(api_yaml_path, 'r', encoding='utf-8') as f:\n proc_spec = yaml.safe_load(f)\n # Merge schemas\n proc_schemas = proc_spec.get('components', {}).get('schemas', {})\n spec_dict['components']['schemas'].update(proc_schemas)\n # Inject processor_config_{name} into WatchBase if the schema is defined\n schema_key = f'processor_config_{proc_name}'\n if schema_key in proc_schemas:\n spec_dict['components']['schemas']['WatchBase']['properties'][schema_key] = {\n '$ref': f'#/components/schemas/{schema_key}'\n }\n # Append x-code-samples from processor paths into existing path operations\n for path, path_item in proc_spec.get('paths', {}).items():\n if path not in spec_dict.get('paths', {}):\n continue\n for method, operation in path_item.items():\n if method not in spec_dict['paths'][path]:\n continue\n if 'x-code-samples' in operation:\n existing = spec_dict['paths'][path][method].get('x-code-samples', [])\n spec_dict['paths'][path][method]['x-code-samples'] = existing + operation['x-code-samples']\n except Exception as e:\n logger.warning(f\"Failed to merge processor API specs: {e}\")\n\n return spec_dict\n\n\n@functools.cache\ndef get_openapi_spec():\n \"\"\"Lazy load OpenAPI spec and dependencies only when validation is needed.\"\"\"\n from openapi_core import OpenAPI # Lazy import - saves ~10.7 MB on startup\n return OpenAPI.from_dict(build_merged_spec_dict())\n\n@functools.cache\ndef get_openapi_schema_dict():\n \"\"\"\n Get the raw OpenAPI spec dictionary for schema access.\n\n Used by Import endpoint to validate and convert query parameters.\n Returns the merged YAML dict (not the OpenAPI object).\n \"\"\"\n return build_merged_spec_dict()\n\n@functools.cache\ndef _resolve_schema_properties(schema_name):\n \"\"\"\n Generic helper to resolve schema properties, including allOf inheritance.\n\n Args:\n schema_name: Name of the schema (e.g., 'WatchBase', 'Watch', 'Tag')\n\n Returns:\n dict: All properties including inherited ones from $ref schemas\n \"\"\"\n spec_dict = get_openapi_schema_dict()\n schema = spec_dict['components']['schemas'].get(schema_name, {})\n\n properties = {}\n\n # Handle allOf (schema inheritance)\n if 'allOf' in schema:\n for item in schema['allOf']:\n # Resolve $ref to parent schema\n if '$ref' in item:\n ref_path = item['$ref'].split('/')[-1]\n ref_schema = spec_dict['components']['schemas'].get(ref_path, {})\n properties.update(ref_schema.get('properties', {}))\n # Add schema-specific properties\n if 'properties' in item:\n properties.update(item['properties'])\n else:\n # Direct properties (no inheritance)\n properties = schema.get('properties', {})\n\n return properties\n\n\n@functools.cache\ndef get_watch_schema_properties():\n \"\"\"\n Extract watch schema properties from OpenAPI spec for Import endpoint.\n\n Returns WatchBase properties (all writable Watch fields).\n \"\"\"\n return _resolve_schema_properties('WatchBase')\n\n# Import readonly field utilities from shared module (avoids circular dependencies with model layer)\nfrom changedetectionio.model.schema_utils import get_readonly_watch_fields, get_readonly_tag_fields\n\n@functools.cache\ndef get_tag_schema_properties():\n \"\"\"\n Extract Tag schema properties from OpenAPI spec.\n\n Returns WatchBase properties + Tag-specific properties (overrides_watch).\n \"\"\"\n return _resolve_schema_properties('Tag')\n\ndef validate_openapi_request(operation_id):\n \"\"\"Decorator to validate incoming requests against OpenAPI spec.\"\"\"\n def decorator(f):\n @functools.wraps(f)\n def wrapper(*args, **kwargs):\n from werkzeug.exceptions import BadRequest\n try:\n # Skip OpenAPI validation for GET requests since they don't have request bodies\n if request.method.upper() != 'GET':\n # Lazy import - only loaded when actually validating a request\n from openapi_core.contrib.flask import FlaskOpenAPIRequest\n from openapi_core.templating.paths.exceptions import ServerNotFound, PathNotFound, PathError\n\n spec = get_openapi_spec()\n openapi_request = FlaskOpenAPIRequest(request)\n result = spec.unmarshal_request(openapi_request)\n if result.errors:\n error_details = []\n for error in result.errors:\n # Skip path/server validation errors for reverse proxy compatibility\n # Flask routing already validates that endpoints exist (returns 404 if not).\n # OpenAPI validation here is primarily for request body schema validation.\n # When behind nginx/reverse proxy, URLs may have path prefixes that don't\n # match the OpenAPI server definitions, causing false positives.\n if isinstance(error, PathError):\n logger.debug(f\"API Call - Skipping path/server validation (delegated to Flask): {error}\")\n continue\n\n error_str = str(error)\n # Extract detailed schema errors from __cause__\n if hasattr(error, '__cause__') and hasattr(error.__cause__, 'schema_errors'):\n for schema_error in error.__cause__.schema_errors:\n field = '.'.join(str(p) for p in schema_error.path) if schema_error.path else 'body'\n msg = schema_error.message if hasattr(schema_error, 'message') else str(schema_error)\n error_details.append(f\"{field}: {msg}\")\n else:\n error_details.append(error_str)\n\n # Only raise if we have actual validation errors (not path/server issues)\n if error_details:\n logger.error(f\"API Call - Validation failed: {'; '.join(error_details)}\")\n raise BadRequest(f\"Validation failed: {'; '.join(error_details)}\")\n except BadRequest:\n # Re-raise BadRequest exceptions (validation failures)\n raise\n except Exception as e:\n # If OpenAPI spec loading fails, log but don't break existing functionality\n logger.critical(f\"OpenAPI validation warning for {operation_id}: {e}\")\n abort(500)\n return f(*args, **kwargs)\n return wrapper\n return decorator\n\n# Import all API resources\nfrom .Watch import Watch, WatchHistory, WatchSingleHistory, WatchHistoryDiff, CreateWatch, WatchFavicon\nfrom .Tags import Tags, Tag\nfrom .Import import Import\nfrom .SystemInfo import SystemInfo\nfrom .Spec import Spec\nfrom .Notifications import Notifications\n\n" }, { "path": "changedetectionio/api/auth.py", "content": "from flask import request, make_response, jsonify\nfrom functools import wraps\n\n\n# Simple API auth key comparison\n# @todo - Maybe short lived token in the future?\n\ndef check_token(f):\n @wraps(f)\n def decorated(*args, **kwargs):\n datastore = args[0].datastore\n\n config_api_token_enabled = datastore.data['settings']['application'].get('api_access_token_enabled')\n config_api_token = datastore.data['settings']['application'].get('api_access_token')\n\n # config_api_token_enabled - a UI option in settings if access should obey the key or not\n if config_api_token_enabled:\n if request.headers.get('x-api-key') != config_api_token:\n return make_response(\n jsonify(\"Invalid access - API key invalid.\"), 403\n )\n\n return f(*args, **kwargs)\n\n return decorated\n" }, { "path": "changedetectionio/auth_decorator.py", "content": "import os\nfrom functools import wraps\nfrom flask import current_app, redirect, request\nfrom loguru import logger\n\ndef login_optionally_required(func):\n \"\"\"\n If password authentication is enabled, verify the user is logged in.\n To be used as a decorator for routes that should optionally require login.\n This version is blueprint-friendly as it uses current_app instead of directly accessing app.\n \"\"\"\n @wraps(func)\n def decorated_view(*args, **kwargs):\n from flask import current_app\n import flask_login\n from flask_login import current_user\n\n # Access datastore through the app config\n datastore = current_app.config['DATASTORE']\n has_password_enabled = datastore.data['settings']['application'].get('password') or os.getenv(\"SALTED_PASS\", False)\n\n # Permitted\n if request.endpoint and 'diff_history_page' in request.endpoint and datastore.data['settings']['application'].get('shared_diff_access'):\n return func(*args, **kwargs)\n elif request.method in flask_login.config.EXEMPT_METHODS:\n return func(*args, **kwargs)\n elif current_app.config.get('LOGIN_DISABLED'):\n return func(*args, **kwargs)\n elif has_password_enabled and not current_user.is_authenticated:\n return current_app.login_manager.unauthorized()\n\n return func(*args, **kwargs)\n return decorated_view" }, { "path": "changedetectionio/blueprint/__init__.py", "content": "" }, { "path": "changedetectionio/blueprint/backups/__init__.py", "content": "import datetime\nimport glob\nimport threading\n\nfrom flask import Blueprint, render_template, send_from_directory, flash, url_for, redirect, abort\nfrom flask_babel import gettext\nimport os\n\nfrom changedetectionio.store import ChangeDetectionStore\nfrom changedetectionio.flask_app import login_optionally_required\nfrom loguru import logger\n\nBACKUP_FILENAME_FORMAT = \"changedetection-backup-{}.zip\"\n\n\ndef create_backup(datastore_path, watches: dict, tags: dict = None):\n logger.debug(\"Creating backup...\")\n import zipfile\n from pathlib import Path\n\n # create a ZipFile object\n timestamp = datetime.datetime.now().strftime(\"%Y%m%d%H%M%S\")\n backupname = BACKUP_FILENAME_FORMAT.format(timestamp)\n backup_filepath = os.path.join(datastore_path, backupname)\n\n with zipfile.ZipFile(backup_filepath.replace('.zip', '.tmp'), \"w\",\n compression=zipfile.ZIP_DEFLATED,\n compresslevel=8) as zipObj:\n\n # Add the settings file (supports both formats)\n # New format: changedetection.json\n changedetection_json = os.path.join(datastore_path, \"changedetection.json\")\n if os.path.isfile(changedetection_json):\n zipObj.write(changedetection_json, arcname=\"changedetection.json\")\n logger.debug(\"Added changedetection.json to backup\")\n\n # Legacy format: url-watches.json (for backward compatibility)\n url_watches_json = os.path.join(datastore_path, \"url-watches.json\")\n if os.path.isfile(url_watches_json):\n zipObj.write(url_watches_json, arcname=\"url-watches.json\")\n logger.debug(\"Added url-watches.json to backup\")\n\n # Add tag data directories (each tag has its own {uuid}/tag.json)\n for uuid, tag in (tags or {}).items():\n for f in Path(tag.data_dir).glob('*'):\n zipObj.write(f,\n arcname=os.path.join(f.parts[-2], f.parts[-1]),\n compress_type=zipfile.ZIP_DEFLATED,\n compresslevel=8)\n logger.debug(f\"Added tag '{tag.get('title')}' ({uuid}) to backup\")\n\n # Add any data in the watch data directory.\n for uuid, w in watches.items():\n for f in Path(w.data_dir).glob('*'):\n zipObj.write(f,\n # Use the full path to access the file, but make the file 'relative' in the Zip.\n arcname=os.path.join(f.parts[-2], f.parts[-1]),\n compress_type=zipfile.ZIP_DEFLATED,\n compresslevel=8)\n\n # Create a list file with just the URLs, so it's easier to port somewhere else in the future\n list_file = \"url-list.txt\"\n with open(os.path.join(datastore_path, list_file), \"w\") as f:\n for uuid in watches:\n url = watches[uuid][\"url\"]\n f.write(\"{}\\r\\n\".format(url))\n list_with_tags_file = \"url-list-with-tags.txt\"\n with open(\n os.path.join(datastore_path, list_with_tags_file), \"w\"\n ) as f:\n for uuid in watches:\n url = watches[uuid].get('url')\n tag = watches[uuid].get('tags', {})\n f.write(\"{} {}\\r\\n\".format(url, tag))\n\n # Add it to the Zip\n zipObj.write(\n os.path.join(datastore_path, list_file),\n arcname=list_file,\n compress_type=zipfile.ZIP_DEFLATED,\n compresslevel=8,\n )\n zipObj.write(\n os.path.join(datastore_path, list_with_tags_file),\n arcname=list_with_tags_file,\n compress_type=zipfile.ZIP_DEFLATED,\n compresslevel=8,\n )\n\n # Now it's done, rename it so it shows up finally and its completed being written.\n os.rename(backup_filepath.replace('.zip', '.tmp'), backup_filepath.replace('.tmp', '.zip'))\n\n\ndef construct_blueprint(datastore: ChangeDetectionStore):\n from .restore import construct_restore_blueprint\n\n backups_blueprint = Blueprint('backups', __name__, template_folder=\"templates\")\n backups_blueprint.register_blueprint(construct_restore_blueprint(datastore))\n backup_threads = []\n\n @login_optionally_required\n @backups_blueprint.route(\"/request-backup\", methods=['GET'])\n def request_backup():\n if any(thread.is_alive() for thread in backup_threads):\n flash(gettext(\"A backup is already running, check back in a few minutes\"), \"error\")\n return redirect(url_for('backups.create'))\n\n if len(find_backups()) > int(os.getenv(\"MAX_NUMBER_BACKUPS\", 100)):\n flash(gettext(\"Maximum number of backups reached, please remove some\"), \"error\")\n return redirect(url_for('backups.create'))\n\n # With immediate persistence, all data is already saved\n zip_thread = threading.Thread(\n target=create_backup,\n args=(datastore.datastore_path, datastore.data.get(\"watching\")),\n kwargs={'tags': datastore.data['settings']['application'].get('tags', {})},\n daemon=True,\n name=\"BackupCreator\"\n )\n zip_thread.start()\n backup_threads.append(zip_thread)\n flash(gettext(\"Backup building in background, check back in a few minutes.\"))\n\n return redirect(url_for('backups.create'))\n\n def find_backups():\n backup_filepath = os.path.join(datastore.datastore_path, BACKUP_FILENAME_FORMAT.format(\"*\"))\n backups = glob.glob(backup_filepath)\n backup_info = []\n\n for backup in backups:\n size = os.path.getsize(backup) / (1024 * 1024)\n creation_time = os.path.getctime(backup)\n backup_info.append({\n 'filename': os.path.basename(backup),\n 'filesize': f\"{size:.2f}\",\n 'creation_time': creation_time\n })\n\n backup_info.sort(key=lambda x: x['creation_time'], reverse=True)\n\n return backup_info\n\n @login_optionally_required\n @backups_blueprint.route(\"/download/\", methods=['GET'])\n def download_backup(filename):\n import re\n filename = filename.strip()\n backup_filename_regex = BACKUP_FILENAME_FORMAT.format(r\"\\d+\")\n\n # Resolve 'latest' before any validation so checks run against the real filename.\n if filename == 'latest':\n backups = find_backups()\n if not backups:\n abort(404)\n filename = backups[0]['filename']\n\n if not re.match(r\"^\" + backup_filename_regex + \"$\", filename):\n abort(400) # Bad Request if the filename doesn't match the pattern\n\n full_path = os.path.join(os.path.abspath(datastore.datastore_path), filename)\n if not full_path.startswith(os.path.abspath(datastore.datastore_path) + os.sep):\n abort(404)\n\n logger.debug(f\"Backup download request for '{full_path}'\")\n return send_from_directory(os.path.abspath(datastore.datastore_path), filename, as_attachment=True)\n\n @login_optionally_required\n @backups_blueprint.route(\"/\", methods=['GET'])\n @backups_blueprint.route(\"/create\", methods=['GET'])\n def create():\n backups = find_backups()\n output = render_template(\"backup_create.html\",\n available_backups=backups,\n backup_running=any(thread.is_alive() for thread in backup_threads)\n )\n return output\n\n @login_optionally_required\n @backups_blueprint.route(\"/remove-backups\", methods=['GET'])\n def remove_backups():\n\n backup_filepath = os.path.join(datastore.datastore_path, BACKUP_FILENAME_FORMAT.format(\"*\"))\n backups = glob.glob(backup_filepath)\n for backup in backups:\n os.unlink(backup)\n\n flash(gettext(\"Backups were deleted.\"))\n\n return redirect(url_for('backups.create'))\n\n return backups_blueprint\n" }, { "path": "changedetectionio/blueprint/backups/restore.py", "content": "import io\nimport json\nimport os\nimport re\nimport shutil\nimport tempfile\nimport threading\nimport zipfile\n\nfrom flask import Blueprint, render_template, flash, url_for, redirect, request\nfrom flask_babel import gettext, lazy_gettext as _l\nfrom wtforms import Form, BooleanField, SubmitField\nfrom flask_wtf.file import FileField, FileAllowed\nfrom loguru import logger\n\nfrom changedetectionio.flask_app import login_optionally_required\n\n# Maximum size of the uploaded zip file. Override via env var MAX_RESTORE_UPLOAD_MB.\n_MAX_UPLOAD_BYTES = int(os.getenv(\"MAX_RESTORE_UPLOAD_MB\", 256)) * 1024 * 1024\n# Maximum total uncompressed size of all entries (zip-bomb guard). Override via MAX_RESTORE_DECOMPRESSED_MB.\n_MAX_DECOMPRESSED_BYTES = int(os.getenv(\"MAX_RESTORE_DECOMPRESSED_MB\", 1024)) * 1024 * 1024\n# Only top-level directories whose name is a valid UUID are treated as watch/tag entries.\n_UUID_RE = re.compile(\n r'^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$',\n re.IGNORECASE,\n)\n\n\nclass RestoreForm(Form):\n zip_file = FileField(_l('Backup zip file'), validators=[\n FileAllowed(['zip'], _l('Must be a .zip backup file!'))\n ])\n include_groups = BooleanField(_l('Include groups'), default=True)\n include_groups_replace_existing = BooleanField(_l('Replace existing groups of the same UUID'), default=True)\n include_watches = BooleanField(_l('Include watches'), default=True)\n include_watches_replace_existing = BooleanField(_l('Replace existing watches of the same UUID'), default=True)\n submit = SubmitField(_l('Restore backup'))\n\n\ndef import_from_zip(zip_stream, datastore, include_groups, include_groups_replace, include_watches, include_watches_replace):\n \"\"\"\n Extract and import watches and groups from a backup zip stream.\n\n Mirrors the store's _load_watches / _load_tags loading pattern:\n - UUID dirs with tag.json → Tag.model + tag_obj.commit()\n - UUID dirs with watch.json → rehydrate_entity + watch_obj.commit()\n\n Returns a dict with counts: restored_groups, skipped_groups, restored_watches, skipped_watches.\n Raises zipfile.BadZipFile if the stream is not a valid zip.\n \"\"\"\n from changedetectionio.model import Tag\n\n restored_groups = 0\n skipped_groups = 0\n restored_watches = 0\n skipped_watches = 0\n\n current_tags = datastore.data['settings']['application'].get('tags', {})\n current_watches = datastore.data['watching']\n\n with tempfile.TemporaryDirectory() as tmpdir:\n logger.debug(f\"Restore: extracting zip to {tmpdir}\")\n with zipfile.ZipFile(zip_stream, 'r') as zf:\n total_uncompressed = sum(m.file_size for m in zf.infolist())\n if total_uncompressed > _MAX_DECOMPRESSED_BYTES:\n raise ValueError(\n f\"Backup archive decompressed size ({total_uncompressed // (1024 * 1024)} MB) \"\n f\"exceeds the {_MAX_DECOMPRESSED_BYTES // (1024 * 1024)} MB limit\"\n )\n resolved_dest = os.path.realpath(tmpdir)\n for member in zf.infolist():\n member_dest = os.path.realpath(os.path.join(resolved_dest, member.filename))\n if not member_dest.startswith(resolved_dest + os.sep) and member_dest != resolved_dest:\n raise ValueError(f\"Zip Slip path traversal detected in backup archive: {member.filename!r}\")\n zf.extract(member, tmpdir)\n logger.debug(\"Restore: zip extracted, scanning UUID directories\")\n\n for entry in os.scandir(tmpdir):\n if not entry.is_dir():\n continue\n\n uuid = entry.name\n if not _UUID_RE.match(uuid):\n logger.warning(f\"Restore: skipping non-UUID directory {uuid!r}\")\n continue\n tag_json_path = os.path.join(entry.path, 'tag.json')\n watch_json_path = os.path.join(entry.path, 'watch.json')\n\n # --- Tags (groups) ---\n if include_groups and os.path.exists(tag_json_path):\n if uuid in current_tags and not include_groups_replace:\n logger.debug(f\"Restore: skipping existing group {uuid} (replace not requested)\")\n skipped_groups += 1\n continue\n\n try:\n with open(tag_json_path, 'r', encoding='utf-8') as f:\n tag_data = json.load(f)\n except (json.JSONDecodeError, IOError) as e:\n logger.error(f\"Restore: failed to read tag.json for {uuid}: {e}\")\n continue\n\n title = tag_data.get('title', uuid)\n logger.debug(f\"Restore: importing group '{title}' ({uuid})\")\n\n # Mirror _load_tags: set uuid and force processor\n tag_data['uuid'] = uuid\n tag_data['processor'] = 'restock_diff'\n\n # Copy the UUID directory so data_dir exists for commit()\n dst_dir = os.path.join(datastore.datastore_path, uuid)\n if os.path.exists(dst_dir):\n shutil.rmtree(dst_dir)\n shutil.copytree(entry.path, dst_dir)\n\n tag_obj = Tag.model(\n datastore_path=datastore.datastore_path,\n __datastore=datastore.data,\n default=tag_data\n )\n current_tags[uuid] = tag_obj\n tag_obj.commit()\n restored_groups += 1\n logger.success(f\"Restore: group '{title}' ({uuid}) restored\")\n\n # --- Watches ---\n elif include_watches and os.path.exists(watch_json_path):\n if uuid in current_watches and not include_watches_replace:\n logger.debug(f\"Restore: skipping existing watch {uuid} (replace not requested)\")\n skipped_watches += 1\n continue\n\n try:\n with open(watch_json_path, 'r', encoding='utf-8') as f:\n watch_data = json.load(f)\n except (json.JSONDecodeError, IOError) as e:\n logger.error(f\"Restore: failed to read watch.json for {uuid}: {e}\")\n continue\n\n url = watch_data.get('url', uuid)\n logger.debug(f\"Restore: importing watch '{url}' ({uuid})\")\n\n # Copy UUID directory first so data_dir and history files exist\n dst_dir = os.path.join(datastore.datastore_path, uuid)\n if os.path.exists(dst_dir):\n shutil.rmtree(dst_dir)\n shutil.copytree(entry.path, dst_dir)\n\n # Mirror _load_watches / rehydrate_entity\n watch_data['uuid'] = uuid\n watch_obj = datastore.rehydrate_entity(uuid, watch_data)\n current_watches[uuid] = watch_obj\n watch_obj.commit()\n restored_watches += 1\n logger.success(f\"Restore: watch '{url}' ({uuid}) restored\")\n\n logger.debug(f\"Restore: scan complete - groups {restored_groups} restored / {skipped_groups} skipped, \"\n f\"watches {restored_watches} restored / {skipped_watches} skipped\")\n\n # Persist changedetection.json (includes the updated tags dict)\n logger.debug(\"Restore: committing datastore settings\")\n datastore.commit()\n\n return {\n 'restored_groups': restored_groups,\n 'skipped_groups': skipped_groups,\n 'restored_watches': restored_watches,\n 'skipped_watches': skipped_watches,\n }\n\n\n\ndef construct_restore_blueprint(datastore):\n restore_blueprint = Blueprint('restore', __name__, template_folder=\"templates\")\n restore_threads = []\n\n @login_optionally_required\n @restore_blueprint.route(\"/restore\", methods=['GET'])\n def restore():\n form = RestoreForm()\n return render_template(\"backup_restore.html\",\n form=form,\n restore_running=any(t.is_alive() for t in restore_threads),\n max_upload_mb=_MAX_UPLOAD_BYTES // (1024 * 1024),\n max_decompressed_mb=_MAX_DECOMPRESSED_BYTES // (1024 * 1024))\n\n @login_optionally_required\n @restore_blueprint.route(\"/restore/start\", methods=['POST'])\n def backups_restore_start():\n if any(t.is_alive() for t in restore_threads):\n flash(gettext(\"A restore is already running, check back in a few minutes\"), \"error\")\n return redirect(url_for('backups.restore.restore'))\n\n zip_file = request.files.get('zip_file')\n if not zip_file or not zip_file.filename:\n flash(gettext(\"No file uploaded\"), \"error\")\n return redirect(url_for('backups.restore.restore'))\n\n if not zip_file.filename.lower().endswith('.zip'):\n flash(gettext(\"File must be a .zip backup file\"), \"error\")\n return redirect(url_for('backups.restore.restore'))\n\n # Reject oversized uploads before reading the stream into memory.\n content_length = request.content_length\n if content_length and content_length > _MAX_UPLOAD_BYTES:\n flash(gettext(\"Backup file is too large (max %(mb)s MB)\", mb=_MAX_UPLOAD_BYTES // (1024 * 1024)), \"error\")\n return redirect(url_for('backups.restore.restore'))\n\n # Read into memory now — the request stream is gone once we return.\n # Read one byte beyond the limit so we can detect truncated-but-still-oversized streams.\n try:\n raw = zip_file.read(_MAX_UPLOAD_BYTES + 1)\n if len(raw) > _MAX_UPLOAD_BYTES:\n flash(gettext(\"Backup file is too large (max %(mb)s MB)\", mb=_MAX_UPLOAD_BYTES // (1024 * 1024)), \"error\")\n return redirect(url_for('backups.restore.restore'))\n zip_bytes = io.BytesIO(raw)\n with zipfile.ZipFile(zip_bytes): # quick validity check before spawning\n pass\n zip_bytes.seek(0)\n except zipfile.BadZipFile:\n flash(gettext(\"Invalid or corrupted zip file\"), \"error\")\n return redirect(url_for('backups.restore.restore'))\n\n include_groups = request.form.get('include_groups') == 'y'\n include_groups_replace = request.form.get('include_groups_replace_existing') == 'y'\n include_watches = request.form.get('include_watches') == 'y'\n include_watches_replace = request.form.get('include_watches_replace_existing') == 'y'\n\n restore_thread = threading.Thread(\n target=import_from_zip,\n kwargs={\n 'zip_stream': zip_bytes,\n 'datastore': datastore,\n 'include_groups': include_groups,\n 'include_groups_replace': include_groups_replace,\n 'include_watches': include_watches,\n 'include_watches_replace': include_watches_replace,\n },\n daemon=True,\n name=\"BackupRestore\"\n )\n restore_thread.start()\n restore_threads[:] = [t for t in restore_threads if t.is_alive()]\n restore_threads.append(restore_thread)\n flash(gettext(\"Restore started in background, check back in a few minutes.\"))\n return redirect(url_for('backups.restore.restore'))\n\n return restore_blueprint\n" }, { "path": "changedetectionio/blueprint/backups/templates/backup_create.html", "content": "{% extends 'base.html' %}\n{% block content %}\n {% from '_helpers.html' import render_simple_field, render_field %}\n\n
\n
\n \n
\n
\n
\n {% if backup_running %}\n

\n  {{ _('A backup is running!') }}\n

\n {% endif %}\n\n

\n {{ _('Here you can download and request a new backup, when a backup is completed you will see it listed below.') }}\n

\n
\n {% if available_backups %}\n
    \n {% for backup in available_backups %}\n
  • \n {{ backup[\"filename\"] }} {{ backup[\"filesize\"] }} {{ _('Mb') }}\n
    • \n {% endfor %}\n
\n {% else %}\n

\n {{ _('No backups found.') }}\n

\n {% endif %}\n\n {{ _('Create backup') }}\n {% if available_backups %}\n {{ _('Remove backups') }}\n {% endif %}\n\n
\n\n
\n
\n{% endblock %}\n" }, { "path": "changedetectionio/blueprint/backups/templates/backup_restore.html", "content": "{% extends 'base.html' %}\n{% block content %}\n {% from '_helpers.html' import render_field, render_checkbox_field %}\n\n
\n
\n \n
\n
\n
\n {% if restore_running %}\n

\n  {{ _('A restore is running!') }}\n

\n {% endif %}\n\n

{{ _('Restore a backup. Must be a .zip backup file created on/after v0.53.1 (new database layout).') }}

\n

{{ _('Note: This does not override the main application settings, only watches and groups.') }}

\n

\n {{ _('Max upload size: %(upload)s MB  ·  Max decompressed size: %(decomp)s MB',\n upload=max_upload_mb, decomp=max_decompressed_mb) }}\n

\n\n
\n \n\n
\n {{ render_checkbox_field(form.include_groups) }}\n {{ _('Include all groups found in backup?') }}\n
\n
\n {{ render_checkbox_field(form.include_groups_replace_existing) }}\n {{ _('Replace any existing groups of the same UUID?') }}\n
\n\n
\n {{ render_checkbox_field(form.include_watches) }}\n {{ _('Include all watches found in backup?') }}\n
\n
\n {{ render_checkbox_field(form.include_watches_replace_existing) }}\n {{ _('Replace any existing watches of the same UUID?') }}\n
\n\n
\n {{ render_field(form.zip_file) }}\n
\n\n
\n \n
\n
\n
\n
\n
\n{% endblock %}\n" }, { "path": "changedetectionio/blueprint/browser_steps/TODO.txt", "content": "- This needs an abstraction to directly handle the puppeteer connection methods\n- Then remove the playwright stuff\n- Remove hack redirect at line 65 changedetectionio/processors/__init__.py\n\nThe screenshots are base64 encoded/decoded which is very CPU intensive for large screenshots (in playwright) but not\nin the direct puppeteer connection (they are binary end to end)\n\n" }, { "path": "changedetectionio/blueprint/browser_steps/__init__.py", "content": "\n# HORRIBLE HACK BUT WORKS :-) PR anyone?\n#\n# Why?\n# `browsersteps_playwright_browser_interface.chromium.connect_over_cdp()` will only run once without async()\n# - this flask app is not async()\n# - A single timeout/keepalive which applies to the session made at .connect_over_cdp()\n#\n# So it means that we must unfortunately for now just keep a single timer since .connect_over_cdp() was run\n# and know when that reaches timeout/keepalive :( when that time is up, restart the connection and tell the user\n# that their time is up, insert another coin. (reload)\n#\n#\n\nfrom changedetectionio.strtobool import strtobool\nfrom flask import Blueprint, request, make_response\nimport os\n\nfrom changedetectionio.store import ChangeDetectionStore\nfrom changedetectionio.flask_app import login_optionally_required\nfrom loguru import logger\n\nbrowsersteps_sessions = {}\nbrowsersteps_watch_to_session = {} # Maps watch_uuid -> browsersteps_session_id\nio_interface_context = None\nimport json\nimport hashlib\nfrom flask import Response\nimport asyncio\nimport threading\nimport time\n\n# Dedicated event loop for ALL browser steps sessions\n_browser_steps_loop = None\n_browser_steps_thread = None\n_browser_steps_loop_lock = threading.Lock()\n\ndef _start_browser_steps_loop():\n \"\"\"Start a dedicated event loop for browser steps in its own thread\"\"\"\n global _browser_steps_loop\n\n # Create and set the event loop for this thread\n loop = asyncio.new_event_loop()\n asyncio.set_event_loop(loop)\n _browser_steps_loop = loop\n\n logger.debug(\"Browser steps event loop started\")\n\n try:\n # Run the loop forever - handles all browsersteps sessions\n loop.run_forever()\n except Exception as e:\n logger.error(f\"Browser steps event loop error: {e}\")\n finally:\n try:\n # Cancel all remaining tasks\n pending = asyncio.all_tasks(loop)\n for task in pending:\n task.cancel()\n\n # Wait for tasks to finish cancellation\n if pending:\n loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True))\n except Exception as e:\n logger.debug(f\"Error during browser steps loop cleanup: {e}\")\n finally:\n loop.close()\n logger.debug(\"Browser steps event loop closed\")\n\ndef _ensure_browser_steps_loop():\n \"\"\"Ensure the browser steps event loop is running\"\"\"\n global _browser_steps_loop, _browser_steps_thread\n\n with _browser_steps_loop_lock:\n if _browser_steps_thread is None or not _browser_steps_thread.is_alive():\n logger.debug(\"Starting browser steps event loop thread\")\n _browser_steps_thread = threading.Thread(\n target=_start_browser_steps_loop,\n daemon=True,\n name=\"BrowserStepsEventLoop\"\n )\n _browser_steps_thread.start()\n\n # Wait for the loop to be ready\n timeout = 5.0\n start_time = time.time()\n while _browser_steps_loop is None:\n if time.time() - start_time > timeout:\n raise RuntimeError(\"Browser steps event loop failed to start\")\n time.sleep(0.01)\n\n logger.debug(\"Browser steps event loop thread started and ready\")\n\ndef run_async_in_browser_loop(coro):\n \"\"\"Run async coroutine using the dedicated browser steps event loop\"\"\"\n _ensure_browser_steps_loop()\n\n if _browser_steps_loop and not _browser_steps_loop.is_closed():\n logger.debug(\"Browser steps using dedicated event loop\")\n future = asyncio.run_coroutine_threadsafe(coro, _browser_steps_loop)\n return future.result()\n else:\n raise RuntimeError(\"Browser steps event loop is not available\")\n\nasync def _close_session_resources(session_data, label=''):\n \"\"\"Close all browser resources for a session in the correct order.\n\n browserstepper.cleanup() closes page+context but not the browser itself.\n For CloakBrowser, browser.close() is what stops the local Chromium process via pw.stop().\n For the default CDP path, playwright_context.stop() shuts down the playwright instance.\n \"\"\"\n browserstepper = session_data.get('browserstepper')\n if browserstepper:\n try:\n await browserstepper.cleanup()\n except Exception as e:\n logger.error(f\"Error cleaning up browserstepper{label}: {e}\")\n\n browser = session_data.get('browser')\n if browser:\n try:\n await asyncio.wait_for(browser.close(), timeout=5.0)\n except Exception as e:\n logger.warning(f\"Error closing browser{label}: {e}\")\n\n playwright_context = session_data.get('playwright_context')\n if playwright_context:\n try:\n await playwright_context.stop()\n except Exception as e:\n logger.warning(f\"Error stopping playwright context{label}: {e}\")\n\n\ndef cleanup_expired_sessions():\n \"\"\"Remove expired browsersteps sessions and cleanup their resources\"\"\"\n global browsersteps_sessions, browsersteps_watch_to_session\n\n expired_session_ids = []\n\n # Find expired sessions\n for session_id, session_data in browsersteps_sessions.items():\n browserstepper = session_data.get('browserstepper')\n if browserstepper and browserstepper.has_expired:\n expired_session_ids.append(session_id)\n\n # Cleanup expired sessions\n for session_id in expired_session_ids:\n logger.debug(f\"Cleaning up expired browsersteps session {session_id}\")\n session_data = browsersteps_sessions[session_id]\n\n try:\n run_async_in_browser_loop(_close_session_resources(session_data, label=f\" for session {session_id}\"))\n except Exception as e:\n logger.error(f\"Error cleaning up session {session_id}: {e}\")\n\n # Remove from sessions dict\n del browsersteps_sessions[session_id]\n\n # Remove from watch mapping\n for watch_uuid, mapped_session_id in list(browsersteps_watch_to_session.items()):\n if mapped_session_id == session_id:\n del browsersteps_watch_to_session[watch_uuid]\n break\n\n if expired_session_ids:\n logger.info(f\"Cleaned up {len(expired_session_ids)} expired browsersteps session(s)\")\n\ndef cleanup_session_for_watch(watch_uuid):\n \"\"\"Cleanup a specific browsersteps session for a watch UUID\"\"\"\n global browsersteps_sessions, browsersteps_watch_to_session\n\n session_id = browsersteps_watch_to_session.get(watch_uuid)\n if not session_id:\n logger.debug(f\"No browsersteps session found for watch {watch_uuid}\")\n return\n\n logger.debug(f\"Cleaning up browsersteps session {session_id} for watch {watch_uuid}\")\n\n session_data = browsersteps_sessions.get(session_id)\n if session_data:\n try:\n run_async_in_browser_loop(_close_session_resources(session_data, label=f\" for watch {watch_uuid}\"))\n except Exception as e:\n logger.error(f\"Error cleaning up session {session_id} for watch {watch_uuid}: {e}\")\n\n # Remove from sessions dict\n del browsersteps_sessions[session_id]\n\n # Remove from watch mapping\n del browsersteps_watch_to_session[watch_uuid]\n\n logger.debug(f\"Cleaned up session for watch {watch_uuid}\")\n\n # Opportunistically cleanup any other expired sessions\n cleanup_expired_sessions()\n\ndef construct_blueprint(datastore: ChangeDetectionStore):\n browser_steps_blueprint = Blueprint('browser_steps', __name__, template_folder=\"templates\")\n\n async def start_browsersteps_session(watch_uuid):\n from changedetectionio.browser_steps import browser_steps\n import time\n from playwright.async_api import async_playwright\n\n keepalive_seconds = int(os.getenv('BROWSERSTEPS_MINUTES_KEEPALIVE', 10)) * 60\n keepalive_ms = ((keepalive_seconds + 3) * 1000)\n\n browsersteps_start_session = {'start_time': time.time()}\n\n # Build proxy dict first — needed by both the CDP path and fetcher-specific launchers\n proxy_id = datastore.get_preferred_proxy_for_watch(uuid=watch_uuid)\n proxy = None\n if proxy_id:\n proxy_url = datastore.proxy_list.get(proxy_id, {}).get('url')\n if proxy_url:\n from urllib.parse import urlparse\n parsed = urlparse(proxy_url)\n proxy = {'server': proxy_url}\n if parsed.username:\n proxy['username'] = parsed.username\n if parsed.password:\n proxy['password'] = parsed.password\n logger.debug(f\"Browser Steps: UUID {watch_uuid} selected proxy {proxy_url}\")\n\n # Resolve the fetcher class for this watch so we can ask it to launch its own browser\n # if it supports that (e.g. CloakBrowser, which runs locally rather than via CDP)\n watch = datastore.data['watching'][watch_uuid]\n from changedetectionio import content_fetchers\n fetcher_name = watch.get_fetch_backend or 'system'\n if fetcher_name == 'system':\n fetcher_name = datastore.data['settings']['application'].get('fetch_backend', 'html_requests')\n fetcher_class = getattr(content_fetchers, fetcher_name, None)\n\n browser = None\n playwright_context = None\n\n # If the fetcher has its own browser launch for the live steps UI, use it.\n # get_browsersteps_browser(proxy, keepalive_ms) returns (browser, playwright_context_or_None)\n # or None to fall back to the default CDP path.\n if fetcher_class and hasattr(fetcher_class, 'get_browsersteps_browser'):\n result = await fetcher_class.get_browsersteps_browser(proxy=proxy, keepalive_ms=keepalive_ms)\n if result is not None:\n browser, playwright_context = result\n logger.debug(f\"Browser Steps: using fetcher-specific browser for '{fetcher_name}'\")\n\n # Default: connect to the remote Playwright/sockpuppetbrowser via CDP\n if browser is None:\n playwright_instance = async_playwright()\n playwright_context = await playwright_instance.start()\n base_url = os.getenv('PLAYWRIGHT_DRIVER_URL', '').strip('\"')\n a = \"?\" if '?' not in base_url else '&'\n base_url += a + f\"timeout={keepalive_ms}\"\n browser = await playwright_context.chromium.connect_over_cdp(base_url, timeout=keepalive_ms)\n logger.debug(f\"Browser Steps: using CDP connection to {base_url}\")\n\n browsersteps_start_session['browser'] = browser\n browsersteps_start_session['playwright_context'] = playwright_context\n\n browserstepper = browser_steps.browsersteps_live_ui(\n playwright_browser=browser,\n proxy=proxy,\n start_url=watch.link,\n headers=watch.get('headers')\n )\n await browserstepper.connect(proxy=proxy)\n browsersteps_start_session['browserstepper'] = browserstepper\n\n return browsersteps_start_session\n\n\n @login_optionally_required\n @browser_steps_blueprint.route(\"/browsersteps_start_session\", methods=['GET'])\n def browsersteps_start_session():\n # A new session was requested, return sessionID\n import uuid\n browsersteps_session_id = str(uuid.uuid4())\n watch_uuid = request.args.get('uuid')\n\n if not watch_uuid:\n return make_response('No Watch UUID specified', 500)\n\n # Cleanup any existing session for this watch\n cleanup_session_for_watch(watch_uuid)\n\n logger.debug(\"Starting connection with playwright\")\n logger.debug(\"browser_steps.py connecting\")\n\n try:\n # Run the async function in the dedicated browser steps event loop\n browsersteps_sessions[browsersteps_session_id] = run_async_in_browser_loop(\n start_browsersteps_session(watch_uuid)\n )\n\n # Store the mapping of watch_uuid -> browsersteps_session_id\n browsersteps_watch_to_session[watch_uuid] = browsersteps_session_id\n\n except Exception as e:\n if 'ECONNREFUSED' in str(e):\n return make_response('Unable to start the Playwright Browser session, is sockpuppetbrowser running? Network configuration is OK?', 401)\n else:\n # Other errors, bad URL syntax, bad reply etc\n return make_response(str(e), 401)\n\n logger.debug(\"Starting connection with playwright - done\")\n return {'browsersteps_session_id': browsersteps_session_id}\n\n @login_optionally_required\n @browser_steps_blueprint.route(\"/browsersteps_image\", methods=['GET'])\n def browser_steps_fetch_screenshot_image():\n from flask import (\n make_response,\n request,\n send_from_directory,\n )\n uuid = request.args.get('uuid')\n step_n = int(request.args.get('step_n'))\n\n watch = datastore.data['watching'].get(uuid)\n filename = f\"step_before-{step_n}.jpeg\" if request.args.get('type', '') == 'before' else f\"step_{step_n}.jpeg\"\n\n if step_n and watch and os.path.isfile(os.path.join(watch.data_dir, filename)):\n response = make_response(send_from_directory(directory=watch.data_dir, path=filename))\n response.headers['Content-type'] = 'image/jpeg'\n response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'\n response.headers['Pragma'] = 'no-cache'\n response.headers['Expires'] = 0\n return response\n\n else:\n return make_response('Unable to fetch image, is the URL correct? does the watch exist? does the step_type-n.jpeg exist?', 401)\n\n # A request for an action was received\n @login_optionally_required\n @browser_steps_blueprint.route(\"/browsersteps_update\", methods=['POST'])\n def browsersteps_ui_update():\n import base64\n\n remaining = 0\n uuid = request.args.get('uuid')\n goto_website_url_first_step = request.args.get('goto_website_url_first_step')\n\n browsersteps_session_id = request.args.get('browsersteps_session_id')\n\n if not browsersteps_session_id:\n return make_response('No browsersteps_session_id specified', 500)\n\n if not browsersteps_sessions.get(browsersteps_session_id):\n return make_response('No session exists under that ID', 500)\n\n is_last_step = False\n\n # @todo - should always be an existing session\n if goto_website_url_first_step:\n logger.debug(\"Going to site (requested automatically before stepping)..\")\n step_operation = \"Goto site\"\n step_selector = None\n step_optional_value = None\n else:\n step_operation = request.form.get('operation')\n step_selector = request.form.get('selector')\n step_optional_value = request.form.get('optional_value')\n is_last_step = strtobool(request.form.get('is_last_step'))\n\n try:\n # Run the async call_action method in the dedicated browser steps event loop\n run_async_in_browser_loop(\n browsersteps_sessions[browsersteps_session_id]['browserstepper'].call_action(\n action_name=step_operation,\n selector=step_selector,\n optional_value=step_optional_value\n )\n )\n\n except Exception as e:\n logger.error(f\"Exception when calling step operation {step_operation} {str(e)}\")\n # Try to find something of value to give back to the user\n return make_response(str(e).splitlines()[0], 401)\n\n # Screenshots and other info only needed on requesting a step (POST)\n try:\n # Run the async get_current_state method in the dedicated browser steps event loop\n (screenshot, xpath_data) = run_async_in_browser_loop(\n browsersteps_sessions[browsersteps_session_id]['browserstepper'].get_current_state()\n )\n\n if is_last_step:\n watch = datastore.data['watching'].get(uuid)\n u = browsersteps_sessions[browsersteps_session_id]['browserstepper'].page.url\n if watch and u:\n watch.save_screenshot(screenshot=screenshot)\n watch.save_xpath_data(data=xpath_data)\n\n except Exception as e:\n return make_response(f\"Error fetching screenshot and element data - {str(e)}\", 401)\n\n # SEND THIS BACK TO THE BROWSER\n output = {\n \"screenshot\": f\"data:image/jpeg;base64,{base64.b64encode(screenshot).decode('ascii')}\",\n \"xpath_data\": xpath_data,\n \"session_age_start\": browsersteps_sessions[browsersteps_session_id]['browserstepper'].age_start,\n \"browser_time_remaining\": round(remaining)\n }\n json_data = json.dumps(output)\n\n # Generate an ETag (hash of the response body)\n etag_hash = hashlib.md5(json_data.encode('utf-8')).hexdigest()\n\n # Create the response with ETag\n response = Response(json_data, mimetype=\"application/json; charset=UTF-8\")\n response.set_etag(etag_hash)\n\n return response\n\n return browser_steps_blueprint\n\n\n" }, { "path": "changedetectionio/blueprint/check_proxies/__init__.py", "content": "import importlib\nfrom concurrent.futures import ThreadPoolExecutor\n\nfrom changedetectionio.processors.text_json_diff.processor import FilterNotFoundInResponse\nfrom changedetectionio.store import ChangeDetectionStore\n\nfrom functools import wraps\n\nfrom flask import Blueprint\nfrom flask_login import login_required\n\nSTATUS_CHECKING = 0\nSTATUS_FAILED = 1\nSTATUS_OK = 2\nTHREADPOOL_MAX_WORKERS = 3\n_DEFAULT_POOL = ThreadPoolExecutor(max_workers=THREADPOOL_MAX_WORKERS)\n\n\n# Maybe use fetch-time if its >5 to show some expected load time?\ndef threadpool(f, executor=None):\n @wraps(f)\n def wrap(*args, **kwargs):\n return (executor or _DEFAULT_POOL).submit(f, *args, **kwargs)\n\n return wrap\n\n\ndef construct_blueprint(datastore: ChangeDetectionStore):\n check_proxies_blueprint = Blueprint('check_proxies', __name__)\n checks_in_progress = {}\n\n @threadpool\n def long_task(uuid, preferred_proxy):\n import time\n from changedetectionio.content_fetchers import exceptions as content_fetcher_exceptions\n from changedetectionio.jinja2_custom import render as jinja_render\n\n status = {'status': '', 'length': 0, 'text': ''}\n\n contents = ''\n now = time.time()\n try:\n processor_module = importlib.import_module(\"changedetectionio.processors.text_json_diff.processor\")\n update_handler = processor_module.perform_site_check(datastore=datastore,\n watch_uuid=uuid\n )\n\n update_handler.call_browser(preferred_proxy_id=preferred_proxy)\n # title, size is len contents not len xfer\n except content_fetcher_exceptions.Non200ErrorCodeReceived as e:\n if e.status_code == 404:\n status.update({'status': 'OK', 'length': len(contents), 'text': f\"OK but 404 (page not found)\"})\n elif e.status_code == 403 or e.status_code == 401:\n status.update({'status': 'ERROR', 'length': len(contents), 'text': f\"{e.status_code} - Access denied\"})\n else:\n status.update({'status': 'ERROR', 'length': len(contents), 'text': f\"Status code: {e.status_code}\"})\n except FilterNotFoundInResponse:\n status.update({'status': 'OK', 'length': len(contents), 'text': f\"OK but CSS/xPath filter not found (page changed layout?)\"})\n except content_fetcher_exceptions.EmptyReply as e:\n if e.status_code == 403 or e.status_code == 401:\n status.update({'status': 'ERROR OTHER', 'length': len(contents), 'text': f\"Got empty reply with code {e.status_code} - Access denied\"})\n else:\n status.update({'status': 'ERROR OTHER', 'length': len(contents) if contents else 0, 'text': f\"Empty reply with code {e.status_code}, needs chrome?\"})\n except content_fetcher_exceptions.ReplyWithContentButNoText as e:\n txt = f\"Got reply but with no content - Status code {e.status_code} - It's possible that the filters were found, but contained no usable text (or contained only an image).\"\n status.update({'status': 'ERROR', 'text': txt})\n except Exception as e:\n status.update({'status': 'ERROR OTHER', 'length': len(contents) if contents else 0, 'text': 'Error: '+type(e).__name__+str(e)})\n else:\n status.update({'status': 'OK', 'length': len(contents), 'text': ''})\n\n if status.get('text'):\n # parse 'text' as text for safety\n v = {'text': status['text']}\n status['text'] = jinja_render(template_str='{{text|e}}', **v)\n\n status['time'] = \"{:.2f}s\".format(time.time() - now)\n\n return status\n\n def _recalc_check_status(uuid):\n\n results = {}\n for k, v in checks_in_progress.get(uuid, {}).items():\n try:\n r_1 = v.result(timeout=0.05)\n except Exception as e:\n # If timeout error?\n results[k] = {'status': 'RUNNING'}\n\n else:\n results[k] = r_1\n\n return results\n\n @login_required\n @check_proxies_blueprint.route(\"//status\", methods=['GET'])\n def get_recheck_status(uuid):\n results = _recalc_check_status(uuid=uuid)\n return results\n\n @login_required\n @check_proxies_blueprint.route(\"//start\", methods=['GET'])\n def start_check(uuid):\n\n if not datastore.proxy_list:\n return\n\n if checks_in_progress.get(uuid):\n state = _recalc_check_status(uuid=uuid)\n for proxy_key, v in state.items():\n if v.get('status') == 'RUNNING':\n return state\n else:\n checks_in_progress[uuid] = {}\n\n for k, v in datastore.proxy_list.items():\n if not checks_in_progress[uuid].get(k):\n checks_in_progress[uuid][k] = long_task(uuid=uuid, preferred_proxy=k)\n\n results = _recalc_check_status(uuid=uuid)\n return results\n\n return check_proxies_blueprint\n" }, { "path": "changedetectionio/blueprint/imports/__init__.py", "content": "from flask import Blueprint, request, redirect, url_for, flash, render_template\nfrom loguru import logger\n\nfrom changedetectionio.store import ChangeDetectionStore\nfrom changedetectionio.auth_decorator import login_optionally_required\n\ndef construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMetaData):\n import_blueprint = Blueprint('imports', __name__, template_folder=\"templates\")\n \n @import_blueprint.route(\"/import\", methods=['GET', 'POST'])\n @login_optionally_required\n def import_page():\n remaining_urls = []\n from changedetectionio import forms\n#\n if request.method == 'POST':\n# from changedetectionio import worker_pool\n\n from changedetectionio.blueprint.imports.importer import (\n import_url_list,\n import_distill_io_json,\n import_xlsx_wachete,\n import_xlsx_custom\n )\n\n # URL List import\n if request.values.get('urls') and len(request.values.get('urls').strip()):\n # Import and push into the queue for immediate update check\n from changedetectionio import processors\n importer_handler = import_url_list()\n importer_handler.run(data=request.values.get('urls'), flash=flash, datastore=datastore, processor=request.values.get('processor', processors.get_default_processor()))\n logger.debug(f\"Imported {len(importer_handler.new_uuids)} new UUIDs\")\n # Dont' add to queue because scheduler can see that they haven't been checked and will add them to the queue\n# for uuid in importer_handler.new_uuids:\n# worker_pool.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))\n\n if len(importer_handler.remaining_data) == 0:\n return redirect(url_for('watchlist.index'))\n else:\n remaining_urls = importer_handler.remaining_data\n\n # Distill.io import\n if request.values.get('distill-io') and len(request.values.get('distill-io').strip()):\n # Import and push into the queue for immediate update check\n d_importer = import_distill_io_json()\n d_importer.run(data=request.values.get('distill-io'), flash=flash, datastore=datastore)\n # Dont' add to queue because scheduler can see that they haven't been checked and will add them to the queue\n# for uuid in importer_handler.new_uuids:\n# worker_pool.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))\n\n\n # XLSX importer\n if request.files and request.files.get('xlsx_file'):\n file = request.files['xlsx_file']\n\n if request.values.get('file_mapping') == 'wachete':\n w_importer = import_xlsx_wachete()\n w_importer.run(data=file, flash=flash, datastore=datastore)\n else:\n w_importer = import_xlsx_custom()\n # Building mapping of col # to col # type\n map = {}\n for i in range(10):\n c = request.values.get(f\"custom_xlsx[col_{i}]\")\n v = request.values.get(f\"custom_xlsx[col_type_{i}]\")\n if c and v:\n map[int(c)] = v\n\n w_importer.import_profile = map\n w_importer.run(data=file, flash=flash, datastore=datastore)\n\n # Dont' add to queue because scheduler can see that they haven't been checked and will add them to the queue\n# for uuid in importer_handler.new_uuids:\n# worker_pool.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))\n\n\n # Could be some remaining, or we could be on GET\n form = forms.importForm(formdata=request.form if request.method == 'POST' else None)\n output = render_template(\"import.html\",\n form=form,\n import_url_list_remaining=\"\\n\".join(remaining_urls),\n original_distill_json=''\n )\n return output\n\n return import_blueprint" }, { "path": "changedetectionio/blueprint/imports/importer.py", "content": "from abc import abstractmethod\nimport time\nfrom wtforms import ValidationError\nfrom loguru import logger\nfrom flask_babel import gettext\n\nfrom changedetectionio.forms import validate_url\n\n\nclass Importer():\n remaining_data = []\n new_uuids = []\n good = 0\n\n def __init__(self):\n self.new_uuids = []\n self.good = 0\n self.remaining_data = []\n self.import_profile = None\n\n @abstractmethod\n def run(self,\n data,\n flash,\n datastore):\n pass\n\n\nclass import_url_list(Importer):\n \"\"\"\n Imports a list, can be in https://example.com tag1, tag2, last tag format\n \"\"\"\n def run(self,\n data,\n flash,\n datastore,\n processor=None\n ):\n\n urls = data.split(\"\\n\")\n good = 0\n now = time.time()\n\n if (len(urls) > 5000):\n flash(gettext(\"Importing 5,000 of the first URLs from your list, the rest can be imported again.\"))\n\n for url in urls:\n url = url.strip()\n if not len(url):\n continue\n\n tags = \"\"\n\n # 'tags' should be a csv list after the URL\n if ' ' in url:\n url, tags = url.split(\" \", 1)\n\n # Flask wtform validators wont work with basic auth, use validators package\n # Up to 5000 per batch so we dont flood the server\n # @todo validators.url will fail when you add your own IP etc\n if len(url) and 'http' in url.lower() and good < 5000:\n extras = None\n if processor:\n extras = {'processor': processor}\n new_uuid = datastore.add_watch(url=url.strip(), tag=tags, save_immediately=False, extras=extras)\n\n if new_uuid:\n # Straight into the queue.\n self.new_uuids.append(new_uuid)\n good += 1\n continue\n\n # Worked past the 'continue' above, append it to the bad list\n if self.remaining_data is None:\n self.remaining_data = []\n self.remaining_data.append(url)\n\n flash(gettext(\"{} Imported from list in {:.2f}s, {} Skipped.\").format(good, time.time() - now, len(self.remaining_data)))\n\n\nclass import_distill_io_json(Importer):\n def run(self,\n data,\n flash,\n datastore,\n ):\n\n import json\n good = 0\n now = time.time()\n self.new_uuids=[]\n\n # @todo Use JSONSchema like in the API to validate here.\n \n try:\n data = json.loads(data.strip())\n except json.decoder.JSONDecodeError:\n flash(gettext(\"Unable to read JSON file, was it broken?\"), 'error')\n return\n\n if not data.get('data'):\n flash(gettext(\"JSON structure looks invalid, was it broken?\"), 'error')\n return\n\n for d in data.get('data'):\n d_config = json.loads(d['config'])\n extras = {'title': d.get('name', None)}\n\n if len(d['uri']) and good < 5000:\n try:\n # @todo we only support CSS ones at the moment\n if d_config['selections'][0]['frames'][0]['excludes'][0]['type'] == 'css':\n extras['subtractive_selectors'] = d_config['selections'][0]['frames'][0]['excludes'][0]['expr']\n except KeyError:\n pass\n except IndexError:\n pass\n extras['include_filters'] = []\n try:\n if d_config['selections'][0]['frames'][0]['includes'][0]['type'] == 'xpath':\n extras['include_filters'].append('xpath:' + d_config['selections'][0]['frames'][0]['includes'][0]['expr'])\n else:\n extras['include_filters'].append(d_config['selections'][0]['frames'][0]['includes'][0]['expr'])\n except KeyError:\n pass\n except IndexError:\n pass\n\n new_uuid = datastore.add_watch(url=d['uri'].strip(),\n tag=\",\".join(d.get('tags', [])),\n extras=extras,\n save_immediately=False)\n\n if new_uuid:\n # Straight into the queue.\n self.new_uuids.append(new_uuid)\n good += 1\n\n flash(gettext(\"{} Imported from Distill.io in {:.2f}s, {} Skipped.\").format(len(self.new_uuids), time.time() - now, len(self.remaining_data)))\n\n\nclass import_xlsx_wachete(Importer):\n\n def run(self,\n data,\n flash,\n datastore,\n ):\n\n good = 0\n now = time.time()\n self.new_uuids = []\n\n from openpyxl import load_workbook\n\n try:\n wb = load_workbook(data)\n except Exception as e:\n # @todo correct except\n flash(gettext(\"Unable to read export XLSX file, something wrong with the file?\"), 'error')\n return\n\n row_id = 2\n for row in wb.active.iter_rows(min_row=row_id):\n try:\n extras = {}\n data = {}\n for cell in row:\n if not cell.value:\n continue\n column_title = wb.active.cell(row=1, column=cell.column).value.strip().lower()\n data[column_title] = cell.value\n\n # Forced switch to webdriver/playwright/etc\n dynamic_wachet = str(data.get('dynamic wachet', '')).strip().lower() # Convert bool to str to cover all cases\n # libreoffice and others can have it as =FALSE() =TRUE(), or bool(true)\n if 'true' in dynamic_wachet or dynamic_wachet == '1':\n extras['fetch_backend'] = 'html_webdriver'\n elif 'false' in dynamic_wachet or dynamic_wachet == '0':\n extras['fetch_backend'] = 'html_requests'\n\n if data.get('xpath'):\n # @todo split by || ?\n extras['include_filters'] = [data.get('xpath')]\n if data.get('name'):\n extras['title'] = data.get('name').strip()\n if data.get('interval (min)'):\n minutes = int(data.get('interval (min)'))\n hours, minutes = divmod(minutes, 60)\n days, hours = divmod(hours, 24)\n weeks, days = divmod(days, 7)\n extras['time_between_check'] = {'weeks': weeks, 'days': days, 'hours': hours, 'minutes': minutes, 'seconds': 0}\n\n # At minimum a URL is required.\n if data.get('url'):\n try:\n validate_url(data.get('url'))\n except ValidationError as e:\n logger.error(f\">> Import URL error {data.get('url')} {str(e)}\")\n flash(gettext(\"Error processing row number {}, URL value was incorrect, row was skipped.\").format(row_id), 'error')\n # Don't bother processing anything else on this row\n continue\n\n new_uuid = datastore.add_watch(url=data['url'].strip(),\n extras=extras,\n tag=data.get('folder'),\n save_immediately=False)\n if new_uuid:\n # Straight into the queue.\n self.new_uuids.append(new_uuid)\n good += 1\n except Exception as e:\n logger.error(e)\n flash(gettext(\"Error processing row number {}, check all cell data types are correct, row was skipped.\").format(row_id), 'error')\n else:\n row_id += 1\n\n flash(gettext(\"{} imported from Wachete .xlsx in {:.2f}s\").format(len(self.new_uuids), time.time() - now))\n\n\nclass import_xlsx_custom(Importer):\n\n def run(self,\n data,\n flash,\n datastore,\n ):\n\n good = 0\n now = time.time()\n self.new_uuids = []\n\n from openpyxl import load_workbook\n\n try:\n wb = load_workbook(data)\n except Exception as e:\n # @todo correct except\n flash(gettext(\"Unable to read export XLSX file, something wrong with the file?\"), 'error')\n return\n\n # @todo cehck atleast 2 rows, same in other method\n from changedetectionio.forms import validate_url\n row_i = 1\n\n try:\n for row in wb.active.iter_rows():\n url = None\n tags = None\n extras = {}\n\n for cell in row:\n if not self.import_profile.get(cell.col_idx):\n continue\n if not cell.value:\n continue\n\n cell_map = self.import_profile.get(cell.col_idx)\n\n cell_val = str(cell.value).strip() # could be bool\n\n if cell_map == 'url':\n url = cell.value.strip()\n try:\n validate_url(url)\n except ValidationError as e:\n logger.error(f\">> Import URL error {url} {str(e)}\")\n flash(gettext(\"Error processing row number {}, URL value was incorrect, row was skipped.\").format(row_i), 'error')\n # Don't bother processing anything else on this row\n url = None\n break\n elif cell_map == 'tag':\n tags = cell.value.strip()\n elif cell_map == 'include_filters':\n # @todo validate?\n extras['include_filters'] = [cell.value.strip()]\n elif cell_map == 'interval_minutes':\n hours, minutes = divmod(int(cell_val), 60)\n days, hours = divmod(hours, 24)\n weeks, days = divmod(days, 7)\n extras['time_between_check'] = {'weeks': weeks, 'days': days, 'hours': hours, 'minutes': minutes, 'seconds': 0}\n else:\n extras[cell_map] = cell_val\n\n # At minimum a URL is required.\n if url:\n new_uuid = datastore.add_watch(url=url,\n extras=extras,\n tag=tags,\n save_immediately=False)\n if new_uuid:\n # Straight into the queue.\n self.new_uuids.append(new_uuid)\n good += 1\n except Exception as e:\n logger.error(e)\n flash(gettext(\"Error processing row number {}, check all cell data types are correct, row was skipped.\").format(row_i), 'error')\n else:\n row_i += 1\n\n flash(gettext(\"{} imported from custom .xlsx in {:.2f}s\").format(len(self.new_uuids), time.time() - now))" }, { "path": "changedetectionio/blueprint/imports/templates/import.html", "content": "{% extends 'base.html' %}\n{% block content %}\n{% from '_helpers.html' import render_field %}\n\n
\n\n
\n \n
\n\n
\n
\n \n
\n\n

\n {{ _('Restoring changedetection.io backups is in the') }} {{ _('backups section') }}.\n
\n

\n
\n {{ _('Enter one URL per line, and optionally add tags for each URL after a space, delineated by comma (,):') }}\n
\n

{{ _('Example:') }} https://example.com tag1, tag2, last tag

\n {{ _('URLs which do not pass validation will stay in the textarea.') }}\n
\n {{ render_field(form.processor, class=\"processor\") }}\n \n
\n \n
\n
\n\n
\n\n
\n
\n {{ _('Copy and Paste your Distill.io watch \\'export\\' file, this should be a JSON file.') }}
\n {{ _('This is') }} {{ _('experimental') }}, {{ _('supported fields are') }} name, uri, tags, config:selections, {{ _('the rest (including') }} schedule) {{ _('are ignored.') }}\n
\n

\n {{ _('How to export?') }} https://distill.io/docs/web-monitor/how-export-and-import-monitors/
\n {{ _('Be sure to set your default fetcher to Chrome if required.') }}
\n

\n
\n \n\n
\n
\n
\n
\n {{ render_field(form.xlsx_file, class=\"processor\") }}\n
\n
\n {{ render_field(form.file_mapping, class=\"processor\") }}\n
\n
\n
\n \n {{ _('Table of custom column and data types mapping for the') }} {{ _('Custom mapping') }} {{ _('File mapping type.') }}\n \n \n \n \n {% for n in range(4) %}\n \n {% endfor %}\n \n \n \n {% for n in range(4) %}\n \n {% endfor %}\n \n
{{ _('Column #') }}
{{ _('Type') }}
\n
\n
\n \n\n
\n\n
\n
\n\n{% endblock %}" }, { "path": "changedetectionio/blueprint/price_data_follower/__init__.py", "content": "\nfrom changedetectionio.strtobool import strtobool\nfrom flask import Blueprint, flash, redirect, url_for\nfrom flask_login import login_required\nfrom changedetectionio.store import ChangeDetectionStore\nfrom changedetectionio import queuedWatchMetaData\nfrom changedetectionio import worker_pool\nfrom queue import PriorityQueue\n\nPRICE_DATA_TRACK_ACCEPT = 'accepted'\nPRICE_DATA_TRACK_REJECT = 'rejected'\n\ndef construct_blueprint(datastore: ChangeDetectionStore, update_q: PriorityQueue):\n\n price_data_follower_blueprint = Blueprint('price_data_follower', __name__)\n\n @login_required\n @price_data_follower_blueprint.route(\"//accept\", methods=['GET'])\n def accept(uuid):\n datastore.data['watching'][uuid]['track_ldjson_price_data'] = PRICE_DATA_TRACK_ACCEPT\n datastore.data['watching'][uuid]['processor'] = 'restock_diff'\n datastore.data['watching'][uuid].clear_watch()\n datastore.data['watching'][uuid].commit()\n worker_pool.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))\n return redirect(url_for(\"watchlist.index\"))\n\n @login_required\n @price_data_follower_blueprint.route(\"//reject\", methods=['GET'])\n def reject(uuid):\n datastore.data['watching'][uuid]['track_ldjson_price_data'] = PRICE_DATA_TRACK_REJECT\n datastore.data['watching'][uuid].commit()\n return redirect(url_for(\"watchlist.index\"))\n\n\n return price_data_follower_blueprint\n\n\n" }, { "path": "changedetectionio/blueprint/rss/__init__.py", "content": "from copy import deepcopy\nfrom loguru import logger\n\nfrom changedetectionio.model import USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH\nfrom changedetectionio.notification import valid_notification_formats\n\nRSS_CONTENT_FORMAT_DEFAULT = 'text'\n\n# Some stuff not related\nRSS_FORMAT_TYPES = deepcopy(valid_notification_formats)\nif RSS_FORMAT_TYPES.get('markdown'):\n del RSS_FORMAT_TYPES['markdown']\n\nif RSS_FORMAT_TYPES.get(USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH):\n del RSS_FORMAT_TYPES[USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH]\n\nif not RSS_FORMAT_TYPES.get(RSS_CONTENT_FORMAT_DEFAULT):\n logger.critical(f\"RSS_CONTENT_FORMAT_DEFAULT not in the acceptable list {RSS_CONTENT_FORMAT_DEFAULT}\")\n\nRSS_TEMPLATE_TYPE_OPTIONS = {'system_default': 'System default', 'notification_body': 'Notification body'}\n\n# @note: We use 
 because nearly all RSS readers render only HTML (Thunderbird for example cant do just plaintext)\nRSS_TEMPLATE_PLAINTEXT_DEFAULT = \"
{{watch_label}} had a change.\\n\\n{{diff}}\\n
\"\n\n# @todo add some [edit]/[history]/[goto] etc links\n# @todo need {{watch_edit_link}} + delete + history link token\nRSS_TEMPLATE_HTML_DEFAULT = \"\\n
{{watch_label}}
\\n
{{diff}}
\\n\\n\"\n"
  },
  {
    "path": "changedetectionio/blueprint/rss/_util.py",
    "content": "\"\"\"\nUtility functions for RSS feed generation.\n\"\"\"\n\nfrom changedetectionio.notification.handler import process_notification\nfrom changedetectionio.notification_service import NotificationContextData, _check_cascading_vars\nfrom loguru import logger\nimport datetime\nimport pytz\nimport re\n\n\nBAD_CHARS_REGEX = r'[\\x00-\\x08\\x0B\\x0C\\x0E-\\x1F]'\n\n\ndef scan_invalid_chars_in_rss(content):\n    \"\"\"\n    Scan for invalid characters in RSS content.\n    Returns True if invalid characters are found.\n    \"\"\"\n    for match in re.finditer(BAD_CHARS_REGEX, content):\n        i = match.start()\n        bad_char = content[i]\n        hex_value = f\"0x{ord(bad_char):02x}\"\n        # Grab context\n        start = max(0, i - 20)\n        end = min(len(content), i + 21)\n        context = content[start:end].replace('\\n', '\\\\n').replace('\\r', '\\\\r')\n        logger.warning(f\"Invalid char {hex_value} at pos {i}: ...{context}...\")\n        # First match is enough\n        return True\n\n    return False\n\n\ndef clean_entry_content(content):\n    \"\"\"\n    Remove invalid characters from RSS content.\n    \"\"\"\n    cleaned = re.sub(BAD_CHARS_REGEX, '', content)\n    return cleaned\n\n\ndef generate_watch_guid(watch, timestamp):\n    \"\"\"\n    Generate a unique GUID for a watch RSS entry.\n\n    Args:\n        watch: The watch object\n        timestamp: The timestamp of the specific change this entry represents\n    \"\"\"\n    return f\"{watch['uuid']}/{timestamp}\"\n\n\ndef validate_rss_token(datastore, request):\n    \"\"\"\n    Validate the RSS access token from the request.\n\n    Returns:\n        tuple: (is_valid, error_response) where error_response is None if valid\n    \"\"\"\n    app_rss_token = datastore.data['settings']['application'].get('rss_access_token')\n    rss_url_token = request.args.get('token')\n\n    if rss_url_token != app_rss_token:\n        return False, (\"Access denied, bad token\", 403)\n\n    return True, None\n\n\ndef get_rss_template(datastore, watch, rss_content_format, default_html, default_plaintext):\n    \"\"\"Get the appropriate template for RSS content.\"\"\"\n    if datastore.data['settings']['application'].get('rss_template_type') == 'notification_body':\n        return _check_cascading_vars(datastore=datastore, var_name='notification_body', watch=watch)\n\n    override = datastore.data['settings']['application'].get('rss_template_override')\n    if override and override.strip():\n        return override\n    elif 'text' in rss_content_format:\n        return default_plaintext\n    else:\n        return default_html\n\n\ndef get_watch_label(datastore, watch):\n    \"\"\"Get the label for a watch based on settings.\"\"\"\n    if datastore.data['settings']['application']['ui'].get('use_page_title_in_list') or watch.get('use_page_title_in_list'):\n        return watch.label\n    else:\n        return watch.get('url')\n\n\ndef add_watch_categories(fe, watch, datastore):\n    \"\"\"Add category tags to a feed entry based on watch tags.\"\"\"\n    for tag_uuid in watch.get('tags', []):\n        tag = datastore.data['settings']['application'].get('tags', {}).get(tag_uuid)\n        if tag and tag.get('title'):\n            fe.category(term=tag.get('title'))\n\n\ndef build_notification_context(watch, timestamp_from, timestamp_to, watch_label,\n                               n_body_template, rss_content_format):\n    \"\"\"Build the notification context object.\"\"\"\n    return NotificationContextData(initial_data={\n        'notification_urls': ['null://just-sending-a-null-test-for-the-render-in-RSS'],\n        'notification_body': n_body_template,\n        'timestamp_to': timestamp_to,\n        'timestamp_from': timestamp_from,\n        'watch_label': watch_label,\n        'notification_format': rss_content_format\n    })\n\n\ndef render_notification(n_object, notification_service, watch, datastore,\n                       date_index_from=None, date_index_to=None):\n    \"\"\"Process and render the notification content.\"\"\"\n    kwargs = {'n_object': n_object, 'watch': watch}\n\n    if date_index_from is not None and date_index_to is not None:\n        kwargs['date_index_from'] = date_index_from\n        kwargs['date_index_to'] = date_index_to\n\n    n_object = notification_service.queue_notification_for_watch(**kwargs)\n    n_object['watch_mime_type'] = None\n\n    res = process_notification(n_object=n_object, datastore=datastore)\n    return res[0]\n\n\ndef populate_feed_entry(fe, watch, content, guid, timestamp, link=None, title_suffix=None):\n    \"\"\"Populate a feed entry with content and metadata.\"\"\"\n    watch_label = watch.get('url')  # Already determined by caller\n\n    # Set link\n    if link:\n        fe.link(link=link)\n\n    # Set title\n    if title_suffix:\n        fe.title(title=f\"{watch_label} - {title_suffix}\")\n    else:\n        fe.title(title=watch_label)\n\n    # Clean and set content\n    if scan_invalid_chars_in_rss(content):\n        content = clean_entry_content(content)\n    fe.content(content=content, type='CDATA')\n\n    # Set GUID\n    fe.guid(guid, permalink=False)\n\n    # Set pubDate using the timestamp of this specific change\n    dt = datetime.datetime.fromtimestamp(int(timestamp))\n    dt = dt.replace(tzinfo=pytz.UTC)\n    fe.pubDate(dt)\n\n"
  },
  {
    "path": "changedetectionio/blueprint/rss/blueprint.py",
    "content": "\nfrom changedetectionio.store import ChangeDetectionStore\nfrom flask import Blueprint\n\nfrom . import tag as tag_routes\nfrom . import main_feed\nfrom . import single_watch\n\ndef construct_blueprint(datastore: ChangeDetectionStore):\n    \"\"\"\n    Construct and configure the RSS blueprint with all routes.\n\n    Args:\n        datastore: The ChangeDetectionStore instance\n\n    Returns:\n        The configured Flask blueprint\n    \"\"\"\n    rss_blueprint = Blueprint('rss', __name__)\n\n    # Register all route modules\n    main_feed.construct_main_feed_routes(rss_blueprint, datastore)\n    single_watch.construct_single_watch_routes(rss_blueprint, datastore)\n    tag_routes.construct_tag_routes(rss_blueprint, datastore)\n\n    return rss_blueprint"
  },
  {
    "path": "changedetectionio/blueprint/rss/main_feed.py",
    "content": "from flask import make_response, request, url_for, redirect\n\n\n\ndef construct_main_feed_routes(rss_blueprint, datastore):\n    \"\"\"\n    Construct the main RSS feed routes.\n\n    Args:\n        rss_blueprint: The Flask blueprint to add routes to\n        datastore: The ChangeDetectionStore instance\n    \"\"\"\n\n    # Some RSS reader situations ended up with rss/ (forward slash after RSS) due\n    # to some earlier blueprint rerouting work, it should goto feed.\n    @rss_blueprint.route(\"/\", methods=['GET'])\n    def extraslash():\n        return redirect(url_for('rss.feed'))\n\n    # Import the login decorator if needed\n    # from changedetectionio.auth_decorator import login_optionally_required\n    @rss_blueprint.route(\"\", methods=['GET'])\n    def feed():\n        from feedgen.feed import FeedGenerator\n        from loguru import logger\n        import time\n\n        from . import RSS_TEMPLATE_HTML_DEFAULT, RSS_TEMPLATE_PLAINTEXT_DEFAULT\n        from ._util import (validate_rss_token, generate_watch_guid, get_rss_template,\n                           get_watch_label, build_notification_context, render_notification,\n                           populate_feed_entry, add_watch_categories)\n        from ...notification_service import NotificationService\n\n        now = time.time()\n\n        # Validate token\n        is_valid, error = validate_rss_token(datastore, request)\n        if not is_valid:\n            return error\n\n        rss_content_format = datastore.data['settings']['application'].get('rss_content_format')\n\n        limit_tag = request.args.get('tag', '').lower().strip()\n        # Be sure limit_tag is a uuid\n        for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():\n            if limit_tag == tag.get('title', '').lower().strip():\n                limit_tag = uuid\n\n        # Sort by last_changed and add the uuid which is usually the key..\n        sorted_watches = []\n\n        # @todo needs a .itemsWithTag() or something - then we can use that in Jinaj2 and throw this away\n        for uuid, watch in datastore.data['watching'].items():\n            # @todo tag notification_muted skip also (improve Watch model)\n            if datastore.data['settings']['application'].get('rss_hide_muted_watches') and watch.get('notification_muted'):\n                continue\n            if limit_tag and not limit_tag in watch['tags']:\n                continue\n            sorted_watches.append(watch)\n\n        sorted_watches.sort(key=lambda x: x.last_changed, reverse=False)\n\n        fg = FeedGenerator()\n        fg.title('changedetection.io')\n        fg.description('Feed description')\n        fg.link(href='https://changedetection.io')\n        notification_service = NotificationService(datastore=datastore, notification_q=False)\n\n        for watch in sorted_watches:\n\n            dates = list(watch.history.keys())\n            # Re #521 - Don't bother processing this one if theres less than 2 snapshots, means we never had a change detected.\n            if len(dates) < 2:\n                continue\n\n            if not watch.viewed:\n                # Re #239 - GUID needs to be individual for each event\n                # @todo In the future make this a configurable link back (see work on BASE_URL https://github.com/dgtlmoon/changedetection.io/pull/228)\n                watch_label = get_watch_label(datastore, watch)\n                timestamp_to = dates[-1]\n                timestamp_from = dates[-2]\n                guid = generate_watch_guid(watch, timestamp_to)\n                # Because we are called via whatever web server, flask should figure out the right path\n                diff_link = {'href': url_for('ui.ui_diff.diff_history_page', uuid=watch['uuid'], _external=True)}\n\n                # Get template and build notification context\n                n_body_template = get_rss_template(datastore, watch, rss_content_format,\n                                                   RSS_TEMPLATE_HTML_DEFAULT, RSS_TEMPLATE_PLAINTEXT_DEFAULT)\n\n                n_object = build_notification_context(watch, timestamp_from, timestamp_to,\n                                                     watch_label, n_body_template, rss_content_format)\n\n                # Render notification\n                res = render_notification(n_object, notification_service, watch, datastore)\n\n                # Create and populate feed entry\n                fe = fg.add_entry()\n                populate_feed_entry(fe, watch, res['body'], guid, timestamp_to, link=diff_link)\n                fe.title(title=watch_label)  # Override title to not include suffix\n                add_watch_categories(fe, watch, datastore)\n\n        response = make_response(fg.rss_str())\n        response.headers.set('Content-Type', 'application/rss+xml;charset=utf-8')\n        logger.trace(f\"RSS generated in {time.time() - now:.3f}s\")\n        return response\n"
  },
  {
    "path": "changedetectionio/blueprint/rss/single_watch.py",
    "content": "\n\ndef construct_single_watch_routes(rss_blueprint, datastore):\n    \"\"\"\n    Construct RSS feed routes for single watches.\n\n    Args:\n        rss_blueprint: The Flask blueprint to add routes to\n        datastore: The ChangeDetectionStore instance\n    \"\"\"\n\n    @rss_blueprint.route(\"/watch/\", methods=['GET'])\n    def rss_single_watch(uuid):\n        import time\n\n        from flask import make_response, request, Response\n        from flask_babel import lazy_gettext as _l\n        from feedgen.feed import FeedGenerator\n        from loguru import logger\n\n        from . import RSS_TEMPLATE_HTML_DEFAULT, RSS_TEMPLATE_PLAINTEXT_DEFAULT\n        from ._util import (validate_rss_token, get_rss_template, get_watch_label,\n                           build_notification_context, render_notification,\n                           populate_feed_entry, add_watch_categories)\n        from ...notification_service import NotificationService\n\n        \"\"\"\n        Display the most recent changes for a single watch as RSS feed.\n        Returns RSS XML with multiple entries showing diffs between consecutive snapshots.\n        The number of entries is controlled by the rss_diff_length setting.\n        \"\"\"\n        now = time.time()\n\n        # Validate token\n        is_valid, error = validate_rss_token(datastore, request)\n        if not is_valid:\n            return error\n\n        rss_content_format = datastore.data['settings']['application'].get('rss_content_format')\n\n        if uuid == 'first':\n            uuid = list(datastore.data['watching'].keys()).pop()\n        # Get the watch by UUID\n        watch = datastore.data['watching'].get(uuid)\n        if not watch:\n            return Response(_l(\"Watch with UUID %(uuid)s not found\", uuid=uuid), status=404, mimetype='text/plain')\n\n        # Check if watch has at least 2 history snapshots\n        dates = list(watch.history.keys())\n        if len(dates) < 2:\n            return Response(_l(\"Watch %(uuid)s does not have enough history snapshots to show changes (need at least 2)\", uuid=uuid), status=400, mimetype='text/plain')\n\n        # Get the number of diffs to include (default: 5)\n        rss_diff_length = datastore.data['settings']['application'].get('rss_diff_length', 5)\n\n        # Calculate how many diffs we can actually show (limited by available history)\n        # We need at least 2 snapshots to create 1 diff\n        max_possible_diffs = len(dates) - 1\n        num_diffs = min(rss_diff_length, max_possible_diffs) if rss_diff_length > 0 else max_possible_diffs\n\n        # Create RSS feed\n        fg = FeedGenerator()\n\n        # Set title: use \"label (url)\" if label differs from url, otherwise just url\n        watch_url = watch.get('url', '')\n        watch_label = get_watch_label(datastore, watch)\n\n        if watch_label != watch_url:\n            feed_title = f'changedetection.io - {watch_label} ({watch_url})'\n        else:\n            feed_title = f'changedetection.io - {watch_url}'\n\n        fg.title(feed_title)\n        fg.description('Changes')\n        fg.link(href='https://changedetection.io')\n\n        # Loop through history and create RSS entries for each diff\n        # Add entries in reverse order because feedgen reverses them\n        # This way, the newest change appears first in the final RSS\n\n        notification_service = NotificationService(datastore=datastore, notification_q=False)\n        for i in range(num_diffs - 1, -1, -1):\n            # Calculate indices for this diff (working backwards from newest)\n            # i=0: compare dates[-2] to dates[-1] (most recent change)\n            # i=1: compare dates[-3] to dates[-2] (previous change)\n            # etc.\n            date_index_to = -(i + 1)\n            date_index_from = -(i + 2)\n            timestamp_to = dates[date_index_to]\n            timestamp_from = dates[date_index_from]\n\n            # Get template and build notification context\n            n_body_template = get_rss_template(datastore, watch, rss_content_format,\n                                               RSS_TEMPLATE_HTML_DEFAULT, RSS_TEMPLATE_PLAINTEXT_DEFAULT)\n\n            n_object = build_notification_context(watch, timestamp_from, timestamp_to,\n                                                 watch_label, n_body_template, rss_content_format)\n\n            # Render notification with date indices\n            res = render_notification(n_object, notification_service, watch, datastore,\n                                     date_index_from, date_index_to)\n\n            # Create and populate feed entry\n            guid = f\"{uuid}/{timestamp_to}\"\n            fe = fg.add_entry()\n            title_suffix = f\"Change @ {res['original_context']['change_datetime']}\"\n            populate_feed_entry(fe, watch, res.get('body', ''), guid, timestamp_to,\n                              link={'href': watch.get('url')}, title_suffix=title_suffix)\n            add_watch_categories(fe, watch, datastore)\n\n        response = make_response(fg.rss_str())\n        response.headers.set('Content-Type', 'application/rss+xml;charset=utf-8')\n        logger.debug(f\"RSS Single watch built in {time.time()-now:.2f}s\")\n\n        return response\n"
  },
  {
    "path": "changedetectionio/blueprint/rss/tag.py",
    "content": "def construct_tag_routes(rss_blueprint, datastore):\n    \"\"\"\n    Construct RSS feed routes for tags.\n\n    Args:\n        rss_blueprint: The Flask blueprint to add routes to\n        datastore: The ChangeDetectionStore instance\n    \"\"\"\n\n    @rss_blueprint.route(\"/tag/\", methods=['GET'])\n    def rss_tag_feed(tag_uuid):\n\n        from flask import make_response, request, url_for\n        from feedgen.feed import FeedGenerator\n\n        from . import RSS_TEMPLATE_HTML_DEFAULT, RSS_TEMPLATE_PLAINTEXT_DEFAULT\n        from ._util import (validate_rss_token, generate_watch_guid, get_rss_template,\n                           get_watch_label, build_notification_context, render_notification,\n                           populate_feed_entry, add_watch_categories)\n        from ...notification_service import NotificationService\n\n        \"\"\"\n        Display an RSS feed for all unviewed watches that belong to a specific tag.\n        Returns RSS XML with entries for each unviewed watch with sufficient history.\n        \"\"\"\n        # Validate token\n        is_valid, error = validate_rss_token(datastore, request)\n        if not is_valid:\n            return error\n\n        rss_content_format = datastore.data['settings']['application'].get('rss_content_format')\n\n        # Verify tag exists\n        tag = datastore.data['settings']['application'].get('tags', {}).get(tag_uuid)\n        if not tag:\n            return f\"Tag with UUID {tag_uuid} not found\", 404\n\n        tag_title = tag.get('title', 'Unknown Tag')\n\n        # Create RSS feed\n        fg = FeedGenerator()\n        fg.title(f'changedetection.io - {tag_title}')\n        fg.description(f'Changes for watches tagged with {tag_title}')\n        fg.link(href='https://changedetection.io')\n        notification_service = NotificationService(datastore=datastore, notification_q=False)\n        # Find all watches with this tag\n        for uuid, watch in datastore.data['watching'].items():\n            #@todo  This is wrong, it needs to sort by most recently changed and then limit it  datastore.data['watching'].items().sorted(?)\n            # So get all watches in this tag then sort\n\n            # Skip if watch doesn't have this tag\n            if tag_uuid not in watch.get('tags', []):\n                continue\n\n            # Skip muted watches if configured\n            if datastore.data['settings']['application'].get('rss_hide_muted_watches') and watch.get('notification_muted'):\n                continue\n\n            # Check if watch has at least 2 history snapshots\n            dates = list(watch.history.keys())\n            if len(dates) < 2:\n                continue\n\n            # Only include unviewed watches\n            if not watch.viewed:\n                # Include a link to the diff page (use uuid from loop, don't modify watch dict)\n                diff_link = {'href': url_for('ui.ui_diff.diff_history_page', uuid=uuid, _external=True)}\n\n                # Get watch label\n                watch_label = get_watch_label(datastore, watch)\n\n                # Get template and build notification context\n                timestamp_to = dates[-1]\n                timestamp_from = dates[-2]\n\n                # Generate GUID for this entry\n                guid = generate_watch_guid(watch, timestamp_to)\n                n_body_template = get_rss_template(datastore, watch, rss_content_format,\n                                                   RSS_TEMPLATE_HTML_DEFAULT, RSS_TEMPLATE_PLAINTEXT_DEFAULT)\n\n                n_object = build_notification_context(watch, timestamp_from, timestamp_to,\n                                                     watch_label, n_body_template, rss_content_format)\n\n                # Render notification\n                res = render_notification(n_object, notification_service, watch, datastore)\n\n                # Create and populate feed entry\n                fe = fg.add_entry()\n                title_suffix = f\"Change @ {res['original_context']['change_datetime']}\"\n                populate_feed_entry(fe, watch, res['body'], guid, timestamp_to, link=diff_link, title_suffix=title_suffix)\n                add_watch_categories(fe, watch, datastore)\n\n        response = make_response(fg.rss_str())\n        response.headers.set('Content-Type', 'application/rss+xml;charset=utf-8')\n        return response\n"
  },
  {
    "path": "changedetectionio/blueprint/settings/__init__.py",
    "content": "import os\nfrom copy import deepcopy\nfrom datetime import datetime, timedelta\nfrom zoneinfo import ZoneInfo, available_timezones\nimport secrets\nimport time\nimport flask_login\nfrom flask import Blueprint, render_template, request, redirect, url_for, flash\nfrom flask_babel import gettext\n\nfrom changedetectionio.store import ChangeDetectionStore\nfrom changedetectionio.auth_decorator import login_optionally_required\n\n\ndef construct_blueprint(datastore: ChangeDetectionStore):\n    settings_blueprint = Blueprint('settings', __name__, template_folder=\"templates\")\n\n    @settings_blueprint.route(\"\", methods=['GET', \"POST\"])\n    @login_optionally_required\n    def settings_page():\n        from changedetectionio import forms\n        from changedetectionio.pluggy_interface import (\n            get_plugin_settings_tabs,\n            load_plugin_settings,\n            save_plugin_settings\n        )\n\n\n        default = deepcopy(datastore.data['settings'])\n        if datastore.proxy_list is not None:\n            available_proxies = list(datastore.proxy_list.keys())\n            # When enabled\n            system_proxy = datastore.data['settings']['requests']['proxy']\n            # In the case it doesnt exist anymore\n            if not system_proxy in available_proxies:\n                system_proxy = None\n\n            default['requests']['proxy'] = system_proxy if system_proxy is not None else available_proxies[0]\n            # Used by the form handler to keep or remove the proxy settings\n            default['proxy_list'] = available_proxies[0]\n\n        # Don't use form.data on POST so that it doesnt overrid the checkbox status from the POST status\n        form = forms.globalSettingsForm(formdata=request.form if request.method == 'POST' else None,\n                                        data=default,\n                                        extra_notification_tokens=datastore.get_unique_notification_tokens_available()\n                                        )\n\n        # Remove the last option 'System default'\n        form.application.form.notification_format.choices.pop()\n\n        if datastore.proxy_list is None:\n            # @todo - Couldn't get setattr() etc dynamic addition working, so remove it instead\n            del form.requests.form.proxy\n        else:\n            form.requests.form.proxy.choices = []\n            for p in datastore.proxy_list:\n                form.requests.form.proxy.choices.append(tuple((p, datastore.proxy_list[p]['label'])))\n\n        if request.method == 'POST':\n            # Password unset is a GET, but we can lock the session to a salted env password to always need the password\n            if form.application.form.data.get('removepassword_button', False):\n                # SALTED_PASS means the password is \"locked\" to what we set in the Env var\n                if not os.getenv(\"SALTED_PASS\", False):\n                    datastore.remove_password()\n                    flash(gettext(\"Password protection removed.\"), 'notice')\n                    flask_login.logout_user()\n                    return redirect(url_for('settings.settings_page'))\n\n            if form.validate():\n                # Don't set password to False when a password is set - should be only removed with the `removepassword` button\n                app_update = dict(deepcopy(form.data['application']))\n\n                # Never update password with '' or False (Added by wtforms when not in submission)\n                if 'password' in app_update and not app_update['password']:\n                    del (app_update['password'])\n\n                datastore.data['settings']['application'].update(app_update)\n\n                # Handle dynamic worker count adjustment\n                old_worker_count = datastore.data['settings']['requests'].get('workers', 1)\n                new_worker_count = form.data['requests'].get('workers', 1)\n\n                datastore.data['settings']['requests'].update(form.data['requests'])\n                datastore.commit()\n\n                # Clear all checksums to force reprocessing with new settings\n                # Global settings can affect watch behavior (filters, rendering, etc.)\n                datastore.clear_all_last_checksums()\n\n                # Adjust worker count if it changed\n                if new_worker_count != old_worker_count:\n                    from changedetectionio import worker_pool\n                    from changedetectionio.flask_app import update_q, notification_q, app, datastore as ds\n\n                    # Check CPU core availability and warn if worker count is high\n                    cpu_count = os.cpu_count()\n                    if cpu_count and new_worker_count >= (cpu_count * 0.9):\n                        flash(gettext(\"Warning: Worker count ({}) is close to or exceeds available CPU cores ({})\").format(\n                            new_worker_count, cpu_count), 'warning')\n\n                    result = worker_pool.adjust_async_worker_count(\n                        new_count=new_worker_count,\n                        update_q=update_q,\n                        notification_q=notification_q,\n                        app=app,\n                        datastore=ds\n                    )\n\n                    if result['status'] == 'success':\n                        flash(gettext(\"Worker count adjusted: {}\").format(result['message']), 'notice')\n                    elif result['status'] == 'not_supported':\n                        flash(gettext(\"Dynamic worker adjustment not supported for sync workers\"), 'warning')\n                    elif result['status'] == 'error':\n                        flash(gettext(\"Error adjusting workers: {}\").format(result['message']), 'error')\n\n                if not os.getenv(\"SALTED_PASS\", False) and len(form.application.form.password.encrypted_password):\n                    datastore.data['settings']['application']['password'] = form.application.form.password.encrypted_password\n                    datastore.commit()\n                    flash(gettext(\"Password protection enabled.\"), 'notice')\n                    flask_login.logout_user()\n                    return redirect(url_for('watchlist.index'))\n\n                # Also save plugin settings from the same form submission\n                plugin_tabs_list = get_plugin_settings_tabs()\n                for tab in plugin_tabs_list:\n                    plugin_id = tab['plugin_id']\n                    form_class = tab['form_class']\n\n                    # Instantiate plugin form with POST data\n                    plugin_form = form_class(formdata=request.form)\n\n                    # Save plugin settings (validation is optional for plugins)\n                    if plugin_form.data:\n                        save_plugin_settings(datastore.datastore_path, plugin_id, plugin_form.data)\n\n                flash(gettext(\"Settings updated.\"))\n\n            else:\n                flash(gettext(\"An error occurred, please see below.\"), \"error\")\n\n        # Convert to ISO 8601 format, all date/time relative events stored as UTC time\n        utc_time = datetime.now(ZoneInfo(\"UTC\")).isoformat()\n\n        # Get active plugins\n        from changedetectionio.pluggy_interface import get_active_plugins\n        import sys\n        active_plugins = get_active_plugins()\n        python_version = f\"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}\"\n\n        # Calculate uptime in seconds\n        uptime_seconds = time.time() - datastore.start_time\n\n        # Get plugin settings tabs and instantiate forms\n        plugin_tabs = get_plugin_settings_tabs()\n        plugin_forms = {}\n\n        for tab in plugin_tabs:\n            plugin_id = tab['plugin_id']\n            form_class = tab['form_class']\n\n            # Load existing settings\n            settings = load_plugin_settings(datastore.datastore_path, plugin_id)\n\n            # Instantiate the form with existing settings\n            plugin_forms[plugin_id] = form_class(data=settings)\n\n        output = render_template(\"settings.html\",\n                                active_plugins=active_plugins,\n                                api_key=datastore.data['settings']['application'].get('api_access_token'),\n                                python_version=python_version,\n                                uptime_seconds=uptime_seconds,\n                                available_timezones=sorted(available_timezones()),\n                                emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),\n                                extra_notification_token_placeholder_info=datastore.get_unique_notification_token_placeholders_available(),\n                                form=form,\n                                hide_remove_pass=os.getenv(\"SALTED_PASS\", False),\n                                min_system_recheck_seconds=int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3)),\n                                settings_application=datastore.data['settings']['application'],\n                                timezone_default_config=datastore.data['settings']['application'].get('scheduler_timezone_default'),\n                                utc_time=utc_time,\n                                plugin_tabs=plugin_tabs,\n                                plugin_forms=plugin_forms,\n                                )\n\n        return output\n\n    @settings_blueprint.route(\"/reset-api-key\", methods=['GET'])\n    @login_optionally_required\n    def settings_reset_api_key():\n        secret = secrets.token_hex(16)\n        datastore.data['settings']['application']['api_access_token'] = secret\n        datastore.commit()\n        flash(gettext(\"API Key was regenerated.\"))\n        return redirect(url_for('settings.settings_page')+'#api')\n        \n    @settings_blueprint.route(\"/notification-logs\", methods=['GET'])\n    @login_optionally_required\n    def notification_logs():\n        from changedetectionio.flask_app import notification_debug_log\n        output = render_template(\"notification-log.html\",\n                               logs=notification_debug_log if len(notification_debug_log) else [\"Notification logs are empty - no notifications sent yet.\"])\n        return output\n\n    @settings_blueprint.route(\"/toggle-all-paused\", methods=['GET'])\n    @login_optionally_required\n    def toggle_all_paused():\n        current_state = datastore.data['settings']['application'].get('all_paused', False)\n        datastore.data['settings']['application']['all_paused'] = not current_state\n        datastore.commit()\n\n        if datastore.data['settings']['application']['all_paused']:\n            flash(gettext(\"Automatic scheduling paused - checks will not be queued.\"), 'notice')\n        else:\n            flash(gettext(\"Automatic scheduling resumed - checks will be queued normally.\"), 'notice')\n\n        return redirect(url_for('watchlist.index'))\n\n    @settings_blueprint.route(\"/toggle-all-muted\", methods=['GET'])\n    @login_optionally_required\n    def toggle_all_muted():\n        current_state = datastore.data['settings']['application'].get('all_muted', False)\n        datastore.data['settings']['application']['all_muted'] = not current_state\n        datastore.commit()\n\n        if datastore.data['settings']['application']['all_muted']:\n            flash(gettext(\"All notifications muted.\"), 'notice')\n        else:\n            flash(gettext(\"All notifications unmuted.\"), 'notice')\n\n        return redirect(url_for('watchlist.index'))\n\n    return settings_blueprint"
  },
  {
    "path": "changedetectionio/blueprint/settings/templates/notification-log.html",
    "content": "{% extends 'base.html' %}\n\n{% block content %}\n
\n     
\n\n         
{{ _('Notification debug log') }}
\n                
\n                
    \n                {% for log in logs|reverse %}\n                    
  • {{log}}
    • \n                {% endfor %}\n                
\n                
\n\n     
\n
\n\n{% endblock %}\n"
  },
  {
    "path": "changedetectionio/blueprint/settings/templates/settings.html",
    "content": "{% extends 'base.html' %}\n\n{% block content %}\n{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form, render_ternary_field, render_fieldlist_with_inline_errors %}\n{% from '_common_fields.html' import render_common_settings_form, show_token_placeholders %}\n\n\n\n\n\n\n\n
\n    
\n        \n    
\n    
\n        
\n            \n            
\n                
\n                    
\n                        {{ render_field(form.requests.form.time_between_check, class=\"time-check-widget\") }}\n\n                        {{ _('Default recheck time for all watches, current system minimum is') }} {{min_system_recheck_seconds}} {{ _('seconds') }} ({{ _('more info') }}).\n                            
\n                                \n                                
\n                                   {{ render_time_schedule_form(form.requests, available_timezones, timezone_default_config) }}\n                                
\n                        
\n                    
\n                    
\n                        {{ render_field(form.application.form.filter_failure_notification_threshold_attempts, class=\"filter_failure_notification_threshold_attempts\") }}\n                        {{ _('After this many consecutive times that the CSS/xPath filter is missing, send a notification') }}\n                            
\n                        {{ _('Set to') }} 0 {{ _('to disable') }}\n                        \n                    
\n                    
\n                        {{ render_field(form.application.form.history_snapshot_max_length, class=\"history_snapshot_max_length\") }}\n                        {{ _('Limit collection of history snapshots for each watch to this number of history items.') }}\n                            
\n                        {{ _('Set to empty to disable / no limit') }}\n                        \n                    
\n\n                    
\n                        {% if not hide_remove_pass %}\n                            {% if current_user.is_authenticated %}\n                                {{ render_button(form.application.form.removepassword_button) }}\n                            {% else %}\n                            {{ render_field(form.application.form.password) }}\n                            {{ _('Password protection for your changedetection.io application.') }}\n                            {% endif %}\n                        {% else %}\n                            {{ _('Password is locked.') }}\n                        {% endif %}\n                    
\n\n                    
\n                        {{ render_checkbox_field(form.application.form.shared_diff_access, class=\"shared_diff_access\") }}\n                        {{ _('Allow access to the watch change history page when password is enabled (Good for sharing the diff page)') }}\n                    
\n                    
\n                        {{ render_checkbox_field(form.application.form.empty_pages_are_a_change) }}\n                        {{ _('When a request returns no content, or the HTML does not contain any text, is this considered a change?') }}\n                    
\n                {% if form.requests.proxy %}\n                    
\n                    
\n                        
\n                            {{ render_field(form.requests.form.proxy, class=\"fetch-backend-proxy\") }}\n                            {{ _('Choose a default proxy for all watches') }}\n                        
\n                    
\n                {% endif %}\n\n                
\n            
\n\n            
\n                
\n                    {{ render_common_settings_form(form.application.form, emailprefix, settings_application, extra_notification_token_placeholder_info) }}\n                
\n                
\n                    {{ render_field(form.application.form.base_url, class=\"m-d\") }}\n                    \n                        {{ _('Base URL used for the') }} {{ '{{ base_url }}' }} {{ _('token in notification links.') }}
\n                        {{ _('Default value is the system environment variable') }} 'BASE_URL' - {{ _('read more here') }}.\n                    \n                
\n            
\n\n            
\n                
\n                    {{ render_field(form.application.form.fetch_backend, class=\"fetch-backend\") }}\n                    \n                        
{{ _('Use the') }} {{ _('Basic') }} {{ _('method (default) where your watched sites don\\'t need Javascript to render.') }}
\n                        
{{ _('The') }} {{ _('Chrome/Javascript') }} {{ _('method requires a network connection to a running WebDriver+Chrome server, set by the ENV var') }} 'WEBDRIVER_URL'. 
\n                    \n                
\n                
\n                    
\n                        {{ _('If you\\'re having trouble waiting for the page to be fully rendered (text missing etc), try increasing the \\'wait\\' time here.') }}\n                        
\n                        {{ _('This will wait') }} n {{ _('seconds before extracting the text.') }}\n                    
\n                    
\n                        {{ render_field(form.application.form.webdriver_delay) }}\n                    
\n                
\n                
\n                    {{ render_field(form.requests.form.workers) }}\n                    {% set worker_info = get_worker_status_info() %}\n                    {{ _('Number of concurrent workers to process watches. More workers = faster processing but higher memory usage.') }}
\n                    {{ _('Currently running:') }} {{ worker_info.count }} {{ _('operational') }} {{ worker_info.type }} {{ _('workers') }}{% if worker_info.active_workers > 0 %} ({{ worker_info.active_workers }} {{ _('actively processing') }}){% endif %}.\n                
\n                
\n                    {{ render_field(form.requests.form.jitter_seconds, class=\"jitter_seconds\") }}\n                    {{ _('Example - 3 seconds random jitter could trigger up to 3 seconds earlier or up to 3 seconds later') }}\n                
\n                
\n                    {{ render_field(form.requests.form.timeout) }}\n                    {{ _('For regular plain requests (not chrome based), maximum number of seconds until timeout, 1-999.') }}
\n                
\n                
\n                    {{ render_field(form.requests.form.default_ua) }}\n                    \n                        {{ _('Applied to all requests.') }}

\n                        {{ _('Note: Simply changing the User-Agent often does not defeat anti-robot technologies, it\\'s important to consider') }} {{ _('all of the ways that the browser is detected') }}.\n                    \n                
\n                
\n                
\n                    {{ _('Tip:') }} {{ _('Connect using Bright Data proxies, find out more here.') }}\n                
\n            
\n\n            
\n\n                    
\n                    {{ render_checkbox_field(form.application.form.ignore_whitespace) }}\n                    {{ _('Ignore whitespace, tabs and new-lines/line-feeds when considering if a change was detected.') }}
\n                    {{ _('Note:') }} {{ _('Changing this will change the status of your existing watches, possibly trigger alerts etc.') }}\n                    \n                    
\n                
\n                    {{ render_checkbox_field(form.application.form.render_anchor_tag_content) }}\n                    {{ _('Render anchor tag content, default disabled, when enabled renders links as') }} (link text)[https://somesite.com]\n                        
\n                    {{ _('Note:') }} {{ _('Changing this could affect the content of your existing watches, possibly trigger alerts etc.') }}\n                    \n                    
\n                    
\n                      {{ render_field(form.application.form.global_subtractive_selectors, rows=5, placeholder=\"header\nfooter\nnav\n.stockticker\n//*[contains(text(), 'Advertisement')]\") }}\n                      \n                        
    \n                          
  •  {{ _('Remove HTML element(s) by CSS and XPath selectors before text conversion.') }} 
    • \n                          
  •  {{ _('Don\\'t paste HTML here, use only CSS and XPath selectors') }} 
    • \n                          
  •  {{ _('Add multiple elements, CSS or XPath selectors per line to ignore multiple parts of the HTML.') }} 
    • \n                        
\n                      \n                    
\n                    
\n                    {{ render_field(form.application.form.global_ignore_text, rows=5, placeholder=\"Some text to ignore in a line\n/some.regex\\d{2}/ for case-INsensitive regex\n                    \") }}\n                    {{ _('Note: This is applied globally in addition to the per-watch rules.') }}
\n                    \n                        
    \n                            
  • {{ _('Matching text will be') }} {{ _('ignored') }} {{ _('in the text snapshot (you can still see it but it wont trigger a change)') }}
    • \n                            
  • {{ _('Note: This is applied globally in addition to the per-watch rules.') }}
    • \n                            
  • {{ _('Each line processed separately, any line matching will be ignored (removed before creating the checksum)') }}
    • \n                            
  • {{ _('Regular Expression support, wrap the entire line in forward slash') }} /regex/
    • \n                            
  • {{ _('Changing this will affect the comparison checksum which may trigger an alert') }}
    • \n                        
\n                     \n                    
\n                    
\n                        {{ render_checkbox_field(form.application.form.strip_ignored_lines) }}\n                        {{ _('Remove any text that appears in the \"Ignore text\" from the output (otherwise its just ignored for change-detection)') }}
\n                        {{ _('Note:') }} {{ _('Changing this will change the status of your existing watches, possibly trigger alerts etc.') }}\n                        \n                    
\n           
\n\n            
\n                
{{ _('API Access') }}
\n                
{{ _('Drive your changedetection.io via API, More about') }} {{ _('API access and examples here') }}.
\n\n                
\n                    {{ render_checkbox_field(form.application.form.api_access_token_enabled) }}\n                    
{{ _('Restrict API access limit by using') }} x-api-key {{ _('header - required for the Chrome Extension to work') }}

\n                    

{{ _('API Key') }} {{api_key}}\n                        {{ _('copy') }}\n                    
\n                
\n                
\n                    {{ _('Regenerate API key') }}\n                
\n                
\n                    
{{ _('Chrome Extension') }}
\n                    
{{ _('Easily add any web-page to your changedetection.io installation from within Chrome.') }}
\n                    {{ _('Step 1') }} {{ _('Install the extension,') }} {{ _('Step 2') }} {{ _('Navigate to this page,') }}\n                    {{ _('Step 3') }} {{ _('Open the extension from the toolbar and click') }} \"{{ _('Sync API Access') }}\"\n                    
\n                        \n                            \"{{\n                            {{ _('Chrome Webstore') }}\n                        \n                    
\n                
\n            
\n            
\n                
\n                    {{ render_checkbox_field(form.application.form.rss_hide_muted_watches) }}\n                
\n                
\n                    {{ render_field(form.application.form.rss_diff_length) }}\n                    {{ _('Maximum number of history snapshots to include in the watch specific RSS feed.') }}\n                
\n                
\n                    {{ render_checkbox_field(form.application.form.rss_reader_mode) }}\n                    {{ _('For watching other RSS feeds - When watching RSS/Atom feeds, convert them into clean text for better change detection.') }}\n                
\n                
\n                    
\n                        {{ render_field(form.application.form.rss_content_format) }}\n                        {{ _('Does your reader support HTML? Set it here') }}\n                    
\n                    
\n                        {{ render_field(form.application.form.rss_template_type) }}\n                        {{ _('\\'System default\\' for the same template for all items, or re-use your \"Notification Body\" as the template.') }}\n                    
\n                    
\n                       {{ render_field(form.application.form.rss_template_override) }}\n                        {{ show_token_placeholders(extra_notification_token_placeholder_info=extra_notification_token_placeholder_info, suffix=\"-rss\") }}\n                    
\n                
\n                
\n\n\n            
\n                
\n                
\n                    {{ _('Ensure the settings below are correct, they are used to manage the time schedule for checking your web page watches.') }}\n                
\n                
\n                    
{{ _('UTC Time & Date from Server:') }} {{ utc_time }}
\n                    
{{ _('Local Time & Date in Browser:') }} 
\n                    
\n                       {{ render_field(form.application.form.scheduler_timezone_default) }}\n                        \n                            {%- for timezone in available_timezones -%}{%- endfor -%}\n                        \n                    
\n                
\n            
\n            
\n                
\n                    {{ render_checkbox_field(form.application.form.ui.form.open_diff_in_new_tab, class=\"open_diff_in_new_tab\") }}\n                    {{ _('Enable this setting to open the diff page in a new tab. If disabled, the diff page will open in the current tab.') }}\n                
\n                
\n                    {{ render_checkbox_field(form.application.form.ui.form.socket_io_enabled, class=\"socket_io_enabled\") }}\n                    {{ _('Realtime UI Updates Enabled - (Restart required if this is changed)') }}\n                
\n                
\n                    {{ render_checkbox_field(form.application.form.ui.form.favicons_enabled, class=\"\") }}\n                    {{ _('Enable or Disable Favicons next to the watch list') }}\n                
\n                
\n                    {{ render_checkbox_field(form.application.form.ui.use_page_title_in_list) }}\n                
\n                
\n                    {{ render_field(form.application.form.pager_size) }}\n                    {{ _('Number of items per page in the watch overview list, 0 to disable.') }}\n                
\n\n            
\n            
\n                
\n                    
\n                        \"BrightData\n                        
BrightData offer world-class proxy services, \"Data Center\" proxies are a very affordable way to proxy your requests, whilst WebUnlocker can help solve most CAPTCHAs.
\n                        
\n                            BrightData offer many many different types of proxies, it is worth reading about what is best for your use-case.\n                        
\n\n                        
\n                            When you have registered, enabled the required services, visit the API example page, then select Python, set the country you wish to use, then copy+paste the access Proxy URL into the \"Extra Proxies\" boxes below.
\n                        
\n                        
\n                            The Proxy URL with BrightData should start with http://brd-customer...\n                        
\n                        
When you sign up using https://brightdata.grsm.io/n0r16zf7eivq BrightData will match any first deposit up to $150
\n                    
\n                    
\n                        \"Oxylabs\n                        
\n                            Collect public data at scale with industry-leading web scraping solutions and the world’s\n                            largest ethical proxy network.\n                        
\n                        
\n                            Oxylabs also provide a WebUnlocker\n                            proxy that bypasses sophisticated anti-bot systems, so you don’t have to.
\n                        
\n                        
\n                            Serve over 195 countries, providing Residential, Mobile and ISP proxies and much more.\n                        
\n                        
\n                            Use the promo code boost35 with this link https://oxylabs.go2cloud.org/SH2d for 35% off Residential, Mobile proxies, Web Unblocker, and Scraper APIs. Built-in proxies enable you to access data from all around the world and help overcome anti-bot solutions.\n\n                        
\n\n                        \n                    
\n                
\n\n                
{{ _('Tip') }}: {{ _('\"Residential\" and \"Mobile\" proxy type can be more successful than \"Data Center\" for blocked websites.') }}
\n\n                
\n                {{ render_fieldlist_with_inline_errors(form.requests.form.extra_proxies) }}\n                {{ _('\"Name\" will be used for selecting the proxy in the Watch Edit settings') }}
\n                {{ _('SOCKS5 proxies with authentication are only supported with \\'plain requests\\' fetcher, for other fetchers you should whitelist the IP access instead') }}\n                
\n                
\n                    
\n                    Extra Browsers can be attached to further defeat CAPTCHA's on websites that are particularly hard to scrape.
\n                    Simply paste the connection address into the box, More instructions and examples here \n                    
\n                    {{ render_fieldlist_with_inline_errors(form.requests.form.extra_browsers) }}\n                
\n            
\n            {% if plugin_tabs %}\n                {% for tab in plugin_tabs %}\n            
\n                {% set plugin_form = plugin_forms[tab.plugin_id] %}\n                {% if tab.template_path %}\n                    {# Plugin provides custom template - include it directly (no separate form) #}\n                    {% include tab.template_path with context %}\n                {% else %}\n                    {# Default form rendering - fields only, no submit button #}\n                    
\n                        {% for field in plugin_form %}\n                            {% if field.type != 'CSRFToken' and field.type != 'SubmitField' %}\n                        
\n                                {% if field.type == 'BooleanField' %}\n                            {{ render_checkbox_field(field) }}\n                                {% else %}\n                            {{ render_field(field) }}\n                                {% endif %}\n                        
\n                            {% endif %}\n                        {% endfor %}\n                    
\n                {% endif %}\n            
\n                {% endfor %}\n            {% endif %}\n            
\n                
{{ _('Uptime:') }} {{ uptime_seconds|format_duration }}
\n                
{{ _('Python version:') }} {{ python_version }}
\n                
{{ _('Plugins active:') }}
\n                {% if active_plugins %}\n                
    \n                    {% for plugin in active_plugins %}\n                    
  • {{ plugin.name }} - {{ plugin.description }}
    • \n                    {% endfor %}\n                
\n                {% else %}\n                
{{ _('No plugins active') }}
\n                {% endif %}\n            
\n            
\n                
\n                    {{ render_button(form.save_button) }}\n                    {{ _('Back') }}\n                    {{ _('Clear Snapshot History') }}\n                
\n            
\n        
\n    
\n
\n\n{% endblock %}\n"
  },
  {
    "path": "changedetectionio/blueprint/tags/README.md",
    "content": "# Groups tags\n\n## How it works\n\nWatch has a list() of tag UUID's, which relate to a config under application.settings.tags\n\nThe 'tag' is actually a watch, because they basically will eventually share 90% of the same config.\n\nSo a tag is like an abstract of a watch\n"
  },
  {
    "path": "changedetectionio/blueprint/tags/__init__.py",
    "content": "import threading\nfrom flask import Blueprint, request, render_template, flash, url_for, redirect\nfrom flask_babel import gettext\nfrom loguru import logger\n\nfrom changedetectionio.store import ChangeDetectionStore\nfrom changedetectionio.flask_app import login_optionally_required\n\n\ndef construct_blueprint(datastore: ChangeDetectionStore):\n    tags_blueprint = Blueprint('tags', __name__, template_folder=\"templates\")\n\n    @tags_blueprint.route(\"/list\", methods=['GET'])\n    @login_optionally_required\n    def tags_overview_page():\n        from .form import SingleTag\n        add_form = SingleTag(request.form)\n\n        sorted_tags = sorted(datastore.data['settings']['application'].get('tags').items(), key=lambda x: x[1]['title'])\n\n        from collections import Counter\n\n        tag_count = Counter(tag for watch in datastore.data['watching'].values() if watch.get('tags') for tag in watch['tags'])\n\n        output = render_template(\"groups-overview.html\",\n                                 app_rss_token=datastore.data['settings']['application'].get('rss_access_token'),\n                                 available_tags=sorted_tags,\n                                 form=add_form,\n                                 tag_count=tag_count,\n                                 )\n\n        return output\n\n    @tags_blueprint.route(\"/add\", methods=['POST'])\n    @login_optionally_required\n    def form_tag_add():\n        from .form import SingleTag\n        add_form = SingleTag(request.form)\n\n        if not add_form.validate():\n            for widget, l in add_form.errors.items():\n                flash(','.join(l), 'error')\n            return redirect(url_for('tags.tags_overview_page'))\n\n        title = request.form.get('name').strip()\n\n        if datastore.tag_exists_by_name(title):\n            flash(gettext('The tag \"{}\" already exists').format(title), \"error\")\n            return redirect(url_for('tags.tags_overview_page'))\n\n        datastore.add_tag(title)\n        flash(gettext(\"Tag added\"))\n\n\n        return redirect(url_for('tags.tags_overview_page'))\n\n    @tags_blueprint.route(\"/mute/\", methods=['GET'])\n    @login_optionally_required\n    def mute(uuid):\n        tag = datastore.data['settings']['application']['tags'].get(uuid)\n        if tag:\n            tag['notification_muted'] = not tag['notification_muted']\n            tag.commit()\n        return redirect(url_for('tags.tags_overview_page'))\n\n    @tags_blueprint.route(\"/delete/\", methods=['GET'])\n    @login_optionally_required\n    def delete(uuid):\n        # Delete the tag from settings immediately\n        if datastore.data['settings']['application']['tags'].get(uuid):\n            del datastore.data['settings']['application']['tags'][uuid]\n\n        # Remove tag from all watches in background thread to avoid blocking\n        def remove_tag_background(tag_uuid):\n            \"\"\"Background thread to remove tag from watches - discarded after completion.\"\"\"\n            removed_count = 0\n            try:\n                for watch_uuid, watch in datastore.data['watching'].items():\n                    if watch.get('tags') and tag_uuid in watch['tags']:\n                        watch['tags'].remove(tag_uuid)\n                        watch.commit()\n                        removed_count += 1\n                logger.info(f\"Background: Tag {tag_uuid} removed from {removed_count} watches\")\n            except Exception as e:\n                logger.error(f\"Error removing tag from watches: {e}\")\n\n        # Start daemon thread\n        threading.Thread(target=remove_tag_background, args=(uuid,), daemon=True).start()\n\n        flash(gettext(\"Tag deleted, removing from watches in background\"))\n        return redirect(url_for('tags.tags_overview_page'))\n\n    @tags_blueprint.route(\"/unlink/\", methods=['GET'])\n    @login_optionally_required\n    def unlink(uuid):\n        # Unlink tag from all watches in background thread to avoid blocking\n        def unlink_tag_background(tag_uuid):\n            \"\"\"Background thread to unlink tag from watches - discarded after completion.\"\"\"\n            unlinked_count = 0\n            try:\n                for watch_uuid, watch in datastore.data['watching'].items():\n                    if watch.get('tags') and tag_uuid in watch['tags']:\n                        watch['tags'].remove(tag_uuid)\n                        watch.commit()\n                        unlinked_count += 1\n                logger.info(f\"Background: Tag {tag_uuid} unlinked from {unlinked_count} watches\")\n            except Exception as e:\n                logger.error(f\"Error unlinking tag from watches: {e}\")\n\n        # Start daemon thread\n        threading.Thread(target=unlink_tag_background, args=(uuid,), daemon=True).start()\n\n        flash(gettext(\"Unlinking tag from watches in background\"))\n        return redirect(url_for('tags.tags_overview_page'))\n\n    @tags_blueprint.route(\"/delete_all\", methods=['GET'])\n    @login_optionally_required\n    def delete_all():\n\n        for tag_uuid in list(datastore.data['settings']['application']['tags'].keys()):\n# TagsDict 'del' handler will remove the dir\n            del datastore.data['settings']['application']['tags'][tag_uuid]\n\n\n        # Clear tags from all watches in background thread to avoid blocking\n        def clear_all_tags_background():\n            \"\"\"Background thread to clear tags from all watches - discarded after completion.\"\"\"\n            cleared_count = 0\n            try:\n                for watch_uuid, watch in datastore.data['watching'].items():\n                    watch['tags'] = []\n                    watch.commit()\n                    cleared_count += 1\n                logger.info(f\"Background: Cleared tags from {cleared_count} watches\")\n            except Exception as e:\n                logger.error(f\"Error clearing tags from watches: {e}\")\n\n        # Start daemon thread\n        threading.Thread(target=clear_all_tags_background, daemon=True).start()\n\n        flash(gettext(\"All tags deleted, clearing from watches in background\"))\n        return redirect(url_for('tags.tags_overview_page'))\n\n    @tags_blueprint.route(\"/edit/\", methods=['GET'])\n    @login_optionally_required\n    def form_tag_edit(uuid):\n        from changedetectionio.blueprint.tags.form import group_restock_settings_form\n        if uuid == 'first':\n            uuid = list(datastore.data['settings']['application']['tags'].keys()).pop()\n\n        default = datastore.data['settings']['application']['tags'].get(uuid)\n        if not default:\n            flash(gettext(\"Tag not found\"), \"error\")\n            return redirect(url_for('watchlist.index'))\n\n        form = group_restock_settings_form(\n                                       formdata=request.form if request.method == 'POST' else None,\n                                       data=default,\n                                       extra_notification_tokens=datastore.get_unique_notification_tokens_available(),\n                                       default_system_settings = datastore.data['settings'],\n                                       )\n\n        # Bridge API-stored processor_config_* values into the form's FormField sub-forms.\n        # The API stores processor_config_restock_diff in the tag dict; find the matching\n        # FormField by checking which one's sub-fields cover the config keys.\n        from wtforms.fields.form import FormField as WTFormField\n        for key, value in default.items():\n            if not key.startswith('processor_config_') or not isinstance(value, dict):\n                continue\n            for form_field in form:\n                if isinstance(form_field, WTFormField) and all(k in form_field.form._fields for k in value):\n                    for sub_key, sub_value in value.items():\n                        sub_field = form_field.form._fields.get(sub_key)\n                        if sub_field is not None:\n                            sub_field.data = sub_value\n                    break\n\n        template_args = {\n            'data': default,\n            'form': form,\n            'watch': default,\n            'extra_notification_token_placeholder_info': datastore.get_unique_notification_token_placeholders_available(),\n        }\n\n        included_content = {}\n        if form.extra_form_content():\n            # So that the extra panels can access _helpers.html etc, we set the environment to load from templates/\n            # And then render the code from the module\n            from jinja2 import Environment, FileSystemLoader\n            import importlib.resources\n            templates_dir = str(importlib.resources.files(\"changedetectionio\").joinpath('templates'))\n            env = Environment(loader=FileSystemLoader(templates_dir))\n            template_str = \"\"\"{% from '_helpers.html' import render_field, render_checkbox_field, render_button %}\n                    \n                
\n                    
\n                        
\n                        {{ render_checkbox_field(form.overrides_watch) }}\n                        Used for watches in \"Restock & Price detection\" mode\n                        
\n                
\n                \"\"\"\n            template_str += form.extra_form_content()\n            template = env.from_string(template_str)\n            included_content = template.render(**template_args)\n\n        output = render_template(\"edit-tag.html\",\n                                 extra_form_content=included_content,\n                                 extra_tab_content=form.extra_tab_content() if form.extra_tab_content() else None,\n                                 settings_application=datastore.data['settings']['application'],\n                                 **template_args\n                                 )\n\n        return output\n\n\n    @tags_blueprint.route(\"/edit/\", methods=['POST'])\n    @login_optionally_required\n    def form_tag_edit_submit(uuid):\n        from changedetectionio.blueprint.tags.form import group_restock_settings_form\n        if uuid == 'first':\n            uuid = list(datastore.data['settings']['application']['tags'].keys()).pop()\n\n        tag = datastore.data['settings']['application']['tags'].get(uuid)\n\n        form = group_restock_settings_form(formdata=request.form if request.method == 'POST' else None,\n                               data=tag,\n                               extra_notification_tokens=datastore.get_unique_notification_tokens_available()\n                               )\n        # @todo subclass form so validation works\n        #if not form.validate():\n#            for widget, l in form.errors.items():\n#                flash(','.join(l), 'error')\n#           return redirect(url_for('tags.form_tag_edit_submit', uuid=uuid))\n\n        tag.update(form.data)\n        tag['processor'] = 'restock_diff'\n        tag.commit()\n\n        # Clear checksums for all watches using this tag to force reprocessing\n        # Tag changes affect inherited configuration\n        cleared_count = datastore.clear_checksums_for_tag(uuid)\n        logger.info(f\"Tag {uuid} updated, cleared {cleared_count} watch checksums\")\n\n        flash(gettext(\"Updated\"))\n\n        return redirect(url_for('tags.tags_overview_page'))\n\n\n    return tags_blueprint\n"
  },
  {
    "path": "changedetectionio/blueprint/tags/form.py",
    "content": "from wtforms import (\n    Form,\n    StringField,\n    SubmitField,\n    validators,\n)\nfrom wtforms.fields.simple import BooleanField\n\nfrom changedetectionio.processors.restock_diff.forms import processor_settings_form as restock_settings_form\n\nclass group_restock_settings_form(restock_settings_form):\n    overrides_watch = BooleanField('Activate for individual watches in this tag/group?', default=False)\n\nclass SingleTag(Form):\n\n    name = StringField('Tag name', [validators.InputRequired()], render_kw={\"placeholder\": \"Name\"})\n    save_button = SubmitField('Save', render_kw={\"class\": \"pure-button pure-button-primary\"})\n\n\n\n\n"
  },
  {
    "path": "changedetectionio/blueprint/tags/templates/edit-tag.html",
    "content": "{% extends 'base.html' %}\n{% block content %}\n{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_ternary_field %}\n{% from '_common_fields.html' import render_common_settings_form %}\n\n\n\n\n\n\n\n\n
\n\n    
\n        \n    
\n\n    
\n        
\n             \n\n            
\n                
\n                    
\n                        {{ render_field(form.title, placeholder=\"https://...\", required=true, class=\"m-d\") }}\n                    
\n                
\n            
\n\n            
\n                
{{ _('These settings are') }} {{ _('added') }} {{ _('to any existing watch configurations.') }}
\n                {% include \"edit/include_subtract.html\" %}\n                
\n                    
{{ _('Text filtering') }}
\n                    {% include \"edit/text-options.html\" %}\n                
\n            
\n\n        {# rendered sub Template #}\n        {% if extra_form_content %}\n            
\n            {{ extra_form_content|safe }}\n            
\n        {% endif %}\n            
\n                
\n                    
\n                      {{ render_ternary_field(form.notification_muted, BooleanField=True) }}\n                    
\n                    {% if 1 %}\n                    
\n                      {{ render_checkbox_field(form.notification_screenshot) }}\n                        \n                            {{ _('Use with caution!') }} {{ _('This will easily fill up your email storage quota or flood other storages.') }}\n                        \n                    
\n                    {% endif %}\n                    
\n                        {% if has_default_notification_urls %}\n                        
\n                            \"{{\n                            {{ _('There are') }} {{ _('system-wide notification URLs enabled') }}, {{ _('this form will override notification settings for this watch only') }} ‐ {{ _('an empty Notification URL list here will still send notifications.') }}\n                        
\n                        {% endif %}\n                        {{ _('Use system defaults') }}\n\n                        {{ render_common_settings_form(form, emailprefix, settings_application, extra_notification_token_placeholder_info) }}\n                    
\n                
\n            
\n\n            
\n                
\n                    {{ render_button(form.save_button) }}\n                
\n            
\n        
\n    
\n
\n\n{% endblock %}\n"
  },
  {
    "path": "changedetectionio/blueprint/tags/templates/groups-overview.html",
    "content": "{% extends 'base.html' %}\n{% block content %}\n{% from '_helpers.html' import render_simple_field, render_field %}\n\n\n\n
\n    
\n        \n        
\n            {{ _('Add a new organisational tag') }}\n            
\n                
\n                    {{ render_simple_field(form.name, placeholder=_(\"Watch group / tag\")) }}\n                
\n                
\n                    {{ render_simple_field(form.save_button, title=_(\"Save\") ) }}\n                
\n            
\n            
\n            
{{ _('Groups allows you to manage filters and notifications for multiple watches under a single organisational tag.') }}
\n        
\n    
\n    \n    
\n\n        \n            \n            \n                \n                \n                \n                \n            \n            \n            \n            \n            {% if not available_tags|length %}\n            \n                \n            \n            {% endif %}\n            {% for uuid, tag in available_tags  %}\n            \n                \n                \n                \n                \n            \n            {% endfor %}\n            \n        
{{ _('# Watches') }}{{ _('Tag / Label name') }}
{{ _('No website organisational tags/groups configured') }}
\n                    \"Mute\n                {{ \"{:,}\".format(tag_count[uuid]) if uuid in tag_count else 0 }} {{ tag.title }}\n                    {{ _('Edit') }}\n                    {{ _('Recheck') }}\n                    Are you sure you want to delete group %(title)s?
This action cannot be undone.
', title=tag.title) }}\"\n                       data-confirm-button=\"{{ _('Delete') }}\"\n                       title=\"{{ _('Deletes and removes tag') }}\">{{ _('Delete') }}\n                    Are you sure you want to unlink all watches from group %(title)s?
The tag will be kept but watches will be removed from it.
', title=tag.title) }}\"\n                       data-confirm-button=\"{{ _('Unlink') }}\"\n                       title=\"{{ _('Keep the tag but unlink any watches') }}\">{{ _('Unlink') }}\n                    \"{{\n                
\n    
\n
\n{% endblock %}\n"
  },
  {
    "path": "changedetectionio/blueprint/ui/__init__.py",
    "content": "import time\nimport threading\nfrom flask import Blueprint, request, redirect, url_for, flash, render_template, session, current_app\nfrom flask_babel import gettext\nfrom loguru import logger\n\nfrom changedetectionio.store import ChangeDetectionStore\nfrom changedetectionio.blueprint.ui.edit import construct_blueprint as construct_edit_blueprint\nfrom changedetectionio.blueprint.ui.notification import construct_blueprint as construct_notification_blueprint\nfrom changedetectionio.blueprint.ui.views import construct_blueprint as construct_views_blueprint\nfrom changedetectionio.blueprint.ui import diff, preview\n\ndef _handle_operations(op, uuids, datastore, worker_pool, update_q, queuedWatchMetaData, watch_check_update, extra_data=None, emit_flash=True):\n    from flask import request, flash\n\n    if op == 'delete':\n        for uuid in uuids:\n            if datastore.data['watching'].get(uuid):\n                datastore.delete(uuid)\n        if emit_flash:\n            flash(gettext(\"{} watches deleted\").format(len(uuids)))\n\n    elif op == 'pause':\n        for uuid in uuids:\n            if datastore.data['watching'].get(uuid):\n                datastore.data['watching'][uuid]['paused'] = True\n                datastore.data['watching'][uuid].commit()\n        if emit_flash:\n            flash(gettext(\"{} watches paused\").format(len(uuids)))\n\n    elif op == 'unpause':\n        for uuid in uuids:\n            if datastore.data['watching'].get(uuid):\n                datastore.data['watching'][uuid.strip()]['paused'] = False\n                datastore.data['watching'][uuid].commit()\n        if emit_flash:\n            flash(gettext(\"{} watches unpaused\").format(len(uuids)))\n\n    elif (op == 'mark-viewed'):\n        for uuid in uuids:\n            if datastore.data['watching'].get(uuid):\n                datastore.set_last_viewed(uuid, int(time.time()))\n        if emit_flash:\n            flash(gettext(\"{} watches updated\").format(len(uuids)))\n\n    elif (op == 'mute'):\n        for uuid in uuids:\n            if datastore.data['watching'].get(uuid):\n                datastore.data['watching'][uuid]['notification_muted'] = True\n                datastore.data['watching'][uuid].commit()\n        if emit_flash:\n            flash(gettext(\"{} watches muted\").format(len(uuids)))\n\n    elif (op == 'unmute'):\n        for uuid in uuids:\n            if datastore.data['watching'].get(uuid):\n                datastore.data['watching'][uuid]['notification_muted'] = False\n                datastore.data['watching'][uuid].commit()\n        if emit_flash:\n            flash(gettext(\"{} watches un-muted\").format(len(uuids)))\n\n    elif (op == 'recheck'):\n        for uuid in uuids:\n            if datastore.data['watching'].get(uuid):\n                # Recheck and require a full reprocessing\n                worker_pool.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))\n        if emit_flash:\n            flash(gettext(\"{} watches queued for rechecking\").format(len(uuids)))\n\n    elif (op == 'clear-errors'):\n        for uuid in uuids:\n            if datastore.data['watching'].get(uuid):\n                datastore.data['watching'][uuid][\"last_error\"] = False\n                datastore.data['watching'][uuid].commit()\n        if emit_flash:\n            flash(gettext(\"{} watches errors cleared\").format(len(uuids)))\n\n    elif (op == 'clear-history'):\n        for uuid in uuids:\n            if datastore.data['watching'].get(uuid):\n                datastore.clear_watch_history(uuid)\n        if emit_flash:\n            flash(gettext(\"{} watches cleared/reset.\").format(len(uuids)))\n\n    elif (op == 'notification-default'):\n        from changedetectionio.notification import (\n            USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH\n        )\n        for uuid in uuids:\n            if datastore.data['watching'].get(uuid):\n                datastore.data['watching'][uuid]['notification_title'] = None\n                datastore.data['watching'][uuid]['notification_body'] = None\n                datastore.data['watching'][uuid]['notification_urls'] = []\n                datastore.data['watching'][uuid]['notification_format'] = USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH\n                datastore.data['watching'][uuid].commit()\n        if emit_flash:\n            flash(gettext(\"{} watches set to use default notification settings\").format(len(uuids)))\n\n    elif (op == 'assign-tag'):\n        op_extradata = extra_data\n        if op_extradata:\n            tag_uuid = datastore.add_tag(title=op_extradata)\n            if op_extradata and tag_uuid:\n                for uuid in uuids:\n                    if datastore.data['watching'].get(uuid):\n                        # Bug in old versions caused by bad edit page/tag handler\n                        if isinstance(datastore.data['watching'][uuid]['tags'], str):\n                            datastore.data['watching'][uuid]['tags'] = []\n\n                        datastore.data['watching'][uuid]['tags'].append(tag_uuid)\n                        datastore.data['watching'][uuid].commit()\n        if emit_flash:\n            flash(gettext(\"{} watches were tagged\").format(len(uuids)))\n\n    if uuids:\n        for uuid in uuids:\n            watch_check_update.send(watch_uuid=uuid)\n\ndef construct_blueprint(datastore: ChangeDetectionStore, update_q, worker_pool, queuedWatchMetaData, watch_check_update):\n    ui_blueprint = Blueprint('ui', __name__, template_folder=\"templates\")\n    \n    # Register the edit blueprint\n    edit_blueprint = construct_edit_blueprint(datastore, update_q, queuedWatchMetaData)\n    ui_blueprint.register_blueprint(edit_blueprint)\n    \n    # Register the notification blueprint\n    notification_blueprint = construct_notification_blueprint(datastore)\n    ui_blueprint.register_blueprint(notification_blueprint)\n    \n    # Register the views blueprint\n    views_blueprint = construct_views_blueprint(datastore, update_q, queuedWatchMetaData, watch_check_update)\n    ui_blueprint.register_blueprint(views_blueprint)\n\n    # Register diff and preview blueprints\n    diff_blueprint = diff.construct_blueprint(datastore)\n    ui_blueprint.register_blueprint(diff_blueprint)\n\n    preview_blueprint = preview.construct_blueprint(datastore)\n    ui_blueprint.register_blueprint(preview_blueprint)\n\n    # Import the login decorator\n    from changedetectionio.auth_decorator import login_optionally_required\n\n    @ui_blueprint.route(\"/clear_history/\", methods=['GET'])\n    @login_optionally_required\n    def clear_watch_history(uuid):\n        try:\n            datastore.clear_watch_history(uuid)\n        except KeyError:\n            flash(gettext('Watch not found'), 'error')\n        else:\n            flash(gettext(\"Cleared snapshot history for watch {}\").format(uuid))\n        return redirect(url_for('watchlist.index'))\n\n    @ui_blueprint.route(\"/clear_history\", methods=['GET', 'POST'])\n    @login_optionally_required\n    def clear_all_history():\n        if request.method == 'POST':\n            confirmtext = request.form.get('confirmtext', '')\n\n            if confirmtext.strip().lower() == gettext('clear').strip().lower():\n                # Run in background thread to avoid blocking\n                def clear_history_background():\n                    # Capture UUIDs first to avoid race conditions\n                    watch_uuids = list(datastore.data['watching'].keys())\n                    logger.info(f\"Background: Clearing history for {len(watch_uuids)} watches\")\n\n                    for uuid in watch_uuids:\n                        try:\n                            datastore.clear_watch_history(uuid)\n                        except Exception as e:\n                            logger.error(f\"Error clearing history for watch {uuid}: {e}\")\n\n                    logger.info(\"Background: Completed clearing history\")\n\n                # Start daemon thread\n                threading.Thread(target=clear_history_background, daemon=True).start()\n\n                flash(gettext(\"History clearing started in background\"))\n            else:\n                flash(gettext('Incorrect confirmation text.'), 'error')\n\n            return redirect(url_for('watchlist.index'))\n\n        output = render_template(\"clear_all_history.html\")\n        return output\n\n    # Clear all statuses, so we do not see the 'unviewed' class\n    @ui_blueprint.route(\"/form/mark-all-viewed\", methods=['GET'])\n    @login_optionally_required\n    def mark_all_viewed():\n        # Save the current newest history as the most recently viewed\n        with_errors = request.args.get('with_errors') == \"1\"\n        tag_limit = request.args.get('tag')\n        now = int(time.time())\n\n        # Mark watches as viewed - use background thread only for large watch counts\n        def mark_viewed_impl():\n            \"\"\"Mark watches as viewed - can run synchronously or in background thread.\"\"\"\n            marked_count = 0\n            try:\n                for watch_uuid, watch in datastore.data['watching'].items():\n                    if with_errors and not watch.get('last_error'):\n                        continue\n\n                    if tag_limit and (not watch.get('tags') or tag_limit not in watch['tags']):\n                        continue\n\n                    datastore.set_last_viewed(watch_uuid, now)\n                    marked_count += 1\n\n                logger.info(f\"Marking complete: {marked_count} watches marked as viewed\")\n            except Exception as e:\n                logger.error(f\"Error marking as viewed: {e}\")\n\n        # For small watch counts (< 10), run synchronously to avoid race conditions in tests\n        # For larger counts, use background thread to avoid blocking the UI\n        watch_count = len(datastore.data['watching'])\n        if watch_count < 10:\n            # Run synchronously for small watch counts\n            mark_viewed_impl()\n        else:\n            # Start background thread for large watch counts\n            thread = threading.Thread(target=mark_viewed_impl, daemon=True)\n            thread.start()\n\n        return redirect(url_for('watchlist.index', tag=tag_limit))\n\n    @ui_blueprint.route(\"/delete\", methods=['GET'])\n    @login_optionally_required\n    def form_delete():\n        uuid = request.args.get('uuid')\n        # More for testing, possible to return the first/only\n        if uuid == 'first':\n            uuid = list(datastore.data['watching'].keys()).pop()\n\n        if uuid != 'all' and not uuid in datastore.data['watching'].keys():\n            flash(gettext('The watch by UUID {} does not exist.').format(uuid), 'error')\n            return redirect(url_for('watchlist.index'))\n\n        datastore.delete(uuid)\n        flash(gettext('Deleted.'))\n\n        return redirect(url_for('watchlist.index'))\n\n    @ui_blueprint.route(\"/clone\", methods=['GET'])\n    @login_optionally_required\n    def form_clone():\n        uuid = request.args.get('uuid')\n\n        if uuid == 'first':\n            uuid = list(datastore.data['watching'].keys()).pop()\n\n        new_uuid = datastore.clone(uuid)\n\n        if not datastore.data['watching'].get(uuid).get('paused'):\n            worker_pool.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=5, item={'uuid': new_uuid}))\n\n        flash(gettext('Cloned, you are editing the new watch.'))\n\n        return redirect(url_for(\"ui.ui_edit.edit_page\", uuid=new_uuid))\n\n    @ui_blueprint.route(\"/checknow\", methods=['GET'])\n    @login_optionally_required\n    def form_watch_checknow():\n        # Forced recheck will skip the 'skip if content is the same' rule (, 'reprocess_existing_data': True})))\n        tag = request.args.get('tag')\n        uuid = request.args.get('uuid')\n        with_errors = request.args.get('with_errors') == \"1\"\n\n        if uuid:\n            # Single watch - check if already queued or running\n            if worker_pool.is_watch_running(uuid) or uuid in update_q.get_queued_uuids():\n                flash(gettext(\"Watch is already queued or being checked.\"))\n            else:\n                worker_pool.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))\n                flash(gettext(\"Queued 1 watch for rechecking.\"))\n        else:\n            # Multiple watches - first count how many need to be queued\n            watches_to_queue = []\n            for k in sorted(datastore.data['watching'].items(), key=lambda item: item[1].get('last_checked', 0)):\n                watch_uuid = k[0]\n                watch = k[1]\n                if not watch['paused'] and watch_uuid:\n                    if with_errors and not watch.get('last_error'):\n                        continue\n                    if tag != None and tag not in watch['tags']:\n                        continue\n                    watches_to_queue.append(watch_uuid)\n\n            # If less than 20 watches, queue synchronously for immediate feedback\n            if len(watches_to_queue) < 20:\n                # Get already queued/running UUIDs once (efficient)\n                queued_uuids = set(update_q.get_queued_uuids())\n                running_uuids = set(worker_pool.get_running_uuids())\n\n                # Filter out watches that are already queued or running\n                watches_to_queue_filtered = []\n                for watch_uuid in watches_to_queue:\n                    if watch_uuid not in queued_uuids and watch_uuid not in running_uuids:\n                        watches_to_queue_filtered.append(watch_uuid)\n\n                # Queue only the filtered watches\n                for watch_uuid in watches_to_queue_filtered:\n                    worker_pool.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid}))\n\n                # Provide feedback about skipped watches\n                skipped_count = len(watches_to_queue) - len(watches_to_queue_filtered)\n                if skipped_count > 0:\n                    flash(gettext(\"Queued {} watches for rechecking ({} already queued or running).\").format(\n                        len(watches_to_queue_filtered), skipped_count))\n                else:\n                    if len(watches_to_queue_filtered) == 1:\n                        flash(gettext(\"Queued 1 watch for rechecking.\"))\n                    else:\n                        flash(gettext(\"Queued {} watches for rechecking.\").format(len(watches_to_queue_filtered)))\n            else:\n                # 20+ watches - queue in background thread to avoid blocking HTTP response\n                # Capture queued/running state before background thread\n                queued_uuids = set(update_q.get_queued_uuids())\n                running_uuids = set(worker_pool.get_running_uuids())\n\n                def queue_watches_background():\n                    \"\"\"Background thread to queue watches - discarded after completion.\"\"\"\n                    try:\n                        queued_count = 0\n                        skipped_count = 0\n                        for watch_uuid in watches_to_queue:\n                            # Check if already queued or running (state captured at start)\n                            if watch_uuid not in queued_uuids and watch_uuid not in running_uuids:\n                                worker_pool.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid}))\n                                queued_count += 1\n                            else:\n                                skipped_count += 1\n\n                        logger.info(f\"Background queueing complete: {queued_count} watches queued, {skipped_count} skipped (already queued/running)\")\n                    except Exception as e:\n                        logger.error(f\"Error in background queueing: {e}\")\n\n                # Start background thread and return immediately\n                thread = threading.Thread(target=queue_watches_background, daemon=True, name=\"QueueWatches-Background\")\n                thread.start()\n\n                # Return immediately with approximate message\n                flash(gettext(\"Queueing watches for rechecking in background...\"))\n\n        return redirect(url_for('watchlist.index', **({'tag': tag} if tag else {})))\n\n    @ui_blueprint.route(\"/form/checkbox-operations\", methods=['POST'])\n    @login_optionally_required\n    def form_watch_list_checkbox_operations():\n        op = request.form['op']\n        uuids = [u.strip() for u in request.form.getlist('uuids') if u]\n        extra_data = request.form.get('op_extradata', '').strip()\n        _handle_operations(\n            datastore=datastore,\n            extra_data=extra_data,\n            queuedWatchMetaData=queuedWatchMetaData,\n            uuids=uuids,\n            worker_pool=worker_pool,\n            update_q=update_q,\n            watch_check_update=watch_check_update,\n            op=op,\n        )\n\n        return redirect(url_for('watchlist.index'))\n\n\n    @ui_blueprint.route(\"/share-url/\", methods=['GET'])\n    @login_optionally_required\n    def form_share_put_watch(uuid):\n        \"\"\"Given a watch UUID, upload the info and return a share-link\n           the share-link can be imported/added\"\"\"\n        import requests\n        import json\n        from copy import deepcopy\n\n\n        # copy it to memory as trim off what we dont need (history)\n        watch = deepcopy(datastore.data['watching'].get(uuid))\n        # For older versions that are not a @property\n        if (watch.get('history')):\n            del (watch['history'])\n\n        # for safety/privacy\n        for k in list(watch.keys()):\n            if k.startswith('notification_'):\n                del watch[k]\n\n        for r in['uuid', 'last_checked', 'last_changed']:\n            if watch.get(r):\n                del (watch[r])\n\n        # Add the global stuff which may have an impact\n        watch['ignore_text'] += datastore.data['settings']['application']['global_ignore_text']\n        watch['subtractive_selectors'] += datastore.data['settings']['application']['global_subtractive_selectors']\n\n        watch_json = json.dumps(watch)\n\n        try:\n            r = requests.request(method=\"POST\",\n                                 data={'watch': watch_json},\n                                 url=\"https://changedetection.io/share/share\",\n                                 headers={'App-Guid': datastore.data['app_guid']})\n            res = r.json()\n\n            # Add to the flask session\n            session['share-link'] = f\"https://changedetection.io/share/{res['share_key']}\"\n\n\n        except Exception as e:\n            logger.error(f\"Error sharing -{str(e)}\")\n            flash(gettext(\"Could not share, something went wrong while communicating with the share server - {}\").format(str(e)), 'error')\n\n        return redirect(url_for('watchlist.index'))\n\n    @ui_blueprint.route(\"/language/auto-detect\", methods=['GET'])\n    def delete_locale_language_session_var_if_it_exists():\n        \"\"\"Clear the session locale preference to auto-detect from browser Accept-Language header\"\"\"\n        if 'locale' in session:\n            session.pop('locale', None)\n            # Refresh Flask-Babel to clear cached locale\n            from flask_babel import refresh\n            refresh()\n            flash(gettext(\"Language set to auto-detect from browser\"))\n\n        # Check if there's a redirect parameter to return to the same page\n        redirect_url = request.args.get('redirect')\n\n        # If redirect is provided and safe, use it\n        from changedetectionio.is_safe_url import is_safe_url\n        if redirect_url and is_safe_url(redirect_url, current_app):\n            return redirect(redirect_url)\n\n        # Otherwise redirect to watchlist\n        return redirect(url_for('watchlist.index'))\n\n    return ui_blueprint"
  },
  {
    "path": "changedetectionio/blueprint/ui/diff.py",
    "content": "from flask import Blueprint, request, redirect, url_for, flash, render_template, make_response, send_from_directory\nfrom flask_babel import gettext\n\nimport re\nimport importlib\nfrom loguru import logger\nfrom markupsafe import Markup\n\nfrom changedetectionio.diff import (\n    REMOVED_STYLE, ADDED_STYLE, REMOVED_INNER_STYLE, ADDED_INNER_STYLE,\n    REMOVED_PLACEMARKER_OPEN, REMOVED_PLACEMARKER_CLOSED,\n    ADDED_PLACEMARKER_OPEN, ADDED_PLACEMARKER_CLOSED,\n    CHANGED_PLACEMARKER_OPEN, CHANGED_PLACEMARKER_CLOSED,\n    CHANGED_INTO_PLACEMARKER_OPEN, CHANGED_INTO_PLACEMARKER_CLOSED\n)\nfrom changedetectionio.store import ChangeDetectionStore\nfrom changedetectionio.auth_decorator import login_optionally_required\n\n\ndef construct_blueprint(datastore: ChangeDetectionStore):\n    diff_blueprint = Blueprint('ui_diff', __name__, template_folder=\"../ui/templates\")\n\n    @diff_blueprint.app_template_filter('diff_unescape_difference_spans')\n    def diff_unescape_difference_spans(content):\n        \"\"\"Emulate Jinja2's auto-escape, then selectively unescape our diff spans.\"\"\"\n        from markupsafe import escape\n\n        if not content:\n            return Markup('')\n\n        # Step 1: Escape everything like Jinja2 would (this makes it XSS-safe)\n        escaped_content = escape(str(content))\n\n        # Step 2: Unescape only our exact diff spans generated by apply_html_color_to_body()\n        # Pattern matches the exact structure:\n        # \n\n        # Unescape outer span opening tags with full attributes (role, aria-label, title)\n        # Matches removed/added/changed/changed_into spans\n        result = re.sub(\n            rf'<span style="({re.escape(REMOVED_STYLE)}|{re.escape(ADDED_STYLE)})" '\n            rf'role="(deletion|insertion|note)" '\n            rf'aria-label="([^&]+?)" '\n            rf'title="([^&]+?)">',\n            r'',\n            str(escaped_content),\n            flags=re.IGNORECASE\n        )\n\n        # Unescape inner span opening tags (without additional attributes)\n        # This matches the darker background styles for changed parts within lines\n        result = re.sub(\n            rf'<span style="({re.escape(REMOVED_INNER_STYLE)}|{re.escape(ADDED_INNER_STYLE)})">',\n            r'',\n            result,\n            flags=re.IGNORECASE\n        )\n\n        # Unescape closing tags (but only as many as we opened)\n        open_count = result.count('', 1)\n\n        return Markup(result)\n\n    @diff_blueprint.route(\"/diff/\", methods=['GET'])\n    @login_optionally_required\n    def diff_history_page(uuid):\n        \"\"\"\n        Render the history/diff page for a watch.\n\n        This route is processor-aware: it delegates rendering to the processor's\n        difference.py module, allowing different processor types to provide\n        custom visualizations:\n        - text_json_diff: Text/HTML diff with syntax highlighting\n        - restock_diff: Could show price charts and stock history\n        - image_diff: Could show image comparison slider/overlay\n\n        Each processor implements processors/{type}/difference.py::render()\n        If a processor doesn't have a difference module, falls back to text_json_diff.\n        \"\"\"\n\n        if uuid == 'first':\n            uuid = list(datastore.data['watching'].keys()).pop()\n\n        try:\n            watch = datastore.data['watching'][uuid]\n        except KeyError:\n            flash(gettext(\"No history found for the specified link, bad link?\"), \"error\")\n            return redirect(url_for('watchlist.index'))\n\n        dates = list(watch.history.keys())\n        if not dates or len(dates) < 2:\n            flash(gettext(\"Not enough history (2 snapshots required) to show difference page for this watch.\"), \"error\")\n            return redirect(url_for('watchlist.index'))\n\n        # Get the processor type for this watch\n        processor_name = watch.get('processor', 'text_json_diff')\n\n        # Try to get the processor's difference module (works for both built-in and plugin processors)\n        from changedetectionio.processors import get_processor_submodule\n        processor_module = get_processor_submodule(processor_name, 'difference')\n\n        # Call the processor's render() function\n        if processor_module and hasattr(processor_module, 'render'):\n            return processor_module.render(\n                watch=watch,\n                datastore=datastore,\n                request=request,\n                url_for=url_for,\n                render_template=render_template,\n                flash=flash,\n                redirect=redirect\n            )\n\n        # Fallback: if processor doesn't have difference module, use text_json_diff as default\n        from changedetectionio.processors.text_json_diff.difference import render as default_render\n        return default_render(\n            watch=watch,\n            datastore=datastore,\n            request=request,\n            url_for=url_for,\n            render_template=render_template,\n            flash=flash,\n            redirect=redirect\n        )\n\n    @diff_blueprint.route(\"/diff//extract\", methods=['GET'])\n    @login_optionally_required\n    def diff_history_page_extract_GET(uuid):\n        \"\"\"\n        Render the data extraction form for a watch.\n\n        This route is processor-aware: it delegates to the processor's\n        extract.py module, allowing different processor types to provide\n        custom extraction interfaces.\n\n        Each processor implements processors/{type}/extract.py::render_form()\n        If a processor doesn't have an extract module, falls back to text_json_diff.\n        \"\"\"\n\n\n        if uuid == 'first':\n            uuid = list(datastore.data['watching'].keys()).pop()\n        try:\n            watch = datastore.data['watching'][uuid]\n        except KeyError:\n            flash(gettext(\"No history found for the specified link, bad link?\"), \"error\")\n            return redirect(url_for('watchlist.index'))\n\n        # Get the processor type for this watch\n        processor_name = watch.get('processor', 'text_json_diff')\n\n        # Try to get the processor's extract module (works for both built-in and plugin processors)\n        from changedetectionio.processors import get_processor_submodule\n        processor_module = get_processor_submodule(processor_name, 'extract')\n\n        # Call the processor's render_form() function\n        if processor_module and hasattr(processor_module, 'render_form'):\n            return processor_module.render_form(\n                watch=watch,\n                datastore=datastore,\n                request=request,\n                url_for=url_for,\n                render_template=render_template,\n                flash=flash,\n                redirect=redirect\n            )\n\n        # Fallback: if processor doesn't have extract module, use base processors.extract as default\n        from changedetectionio.processors.extract import render_form as default_render_form\n        return default_render_form(\n            watch=watch,\n            datastore=datastore,\n            request=request,\n            url_for=url_for,\n            render_template=render_template,\n            flash=flash,\n            redirect=redirect\n        )\n\n    @diff_blueprint.route(\"/diff//extract\", methods=['POST'])\n    @login_optionally_required\n    def diff_history_page_extract_POST(uuid):\n        \"\"\"\n        Process the data extraction request.\n\n        This route is processor-aware: it delegates to the processor's\n        extract.py module, allowing different processor types to provide\n        custom extraction logic.\n\n        Each processor implements processors/{type}/extract.py::process_extraction()\n        If a processor doesn't have an extract module, falls back to text_json_diff.\n        \"\"\"\n\n        if uuid == 'first':\n            uuid = list(datastore.data['watching'].keys()).pop()\n\n        try:\n            watch = datastore.data['watching'][uuid]\n        except KeyError:\n            flash(gettext(\"No history found for the specified link, bad link?\"), \"error\")\n            return redirect(url_for('watchlist.index'))\n\n        # Get the processor type for this watch\n        processor_name = watch.get('processor', 'text_json_diff')\n\n        # Try to get the processor's extract module (works for both built-in and plugin processors)\n        from changedetectionio.processors import get_processor_submodule\n        processor_module = get_processor_submodule(processor_name, 'extract')\n\n        # Call the processor's process_extraction() function\n        if processor_module and hasattr(processor_module, 'process_extraction'):\n            return processor_module.process_extraction(\n                watch=watch,\n                datastore=datastore,\n                request=request,\n                url_for=url_for,\n                make_response=make_response,\n                send_from_directory=send_from_directory,\n                flash=flash,\n                redirect=redirect\n            )\n\n        # Fallback: if processor doesn't have extract module, use base processors.extract as default\n        from changedetectionio.processors.extract import process_extraction as default_process_extraction\n        return default_process_extraction(\n            watch=watch,\n            datastore=datastore,\n            request=request,\n            url_for=url_for,\n            make_response=make_response,\n            send_from_directory=send_from_directory,\n            flash=flash,\n            redirect=redirect\n        )\n\n    @diff_blueprint.route(\"/diff//processor-asset/\", methods=['GET'])\n    @login_optionally_required\n    def processor_asset(uuid, asset_name):\n        \"\"\"\n        Serve processor-specific binary assets (images, files, etc.).\n\n        This route is processor-aware: it delegates to the processor's\n        difference.py module, allowing different processor types to serve\n        custom assets without embedding them as base64 in templates.\n\n        This solves memory issues with large binary data (e.g., screenshots)\n        by streaming them as separate HTTP responses instead of embedding\n        in the HTML template.\n\n        Each processor implements processors/{type}/difference.py::get_asset()\n        which returns (binary_data, content_type, cache_control_header).\n\n        Example URLs:\n        - /diff/{uuid}/processor-asset/before\n        - /diff/{uuid}/processor-asset/after\n        - /diff/{uuid}/processor-asset/rendered_diff\n        \"\"\"\n\n        if uuid == 'first':\n            uuid = list(datastore.data['watching'].keys()).pop()\n\n        try:\n            watch = datastore.data['watching'][uuid]\n        except KeyError:\n            flash(gettext(\"No history found for the specified link, bad link?\"), \"error\")\n            return redirect(url_for('watchlist.index'))\n\n        # Get the processor type for this watch\n        processor_name = watch.get('processor', 'text_json_diff')\n\n        # Try to get the processor's difference module (works for both built-in and plugin processors)\n        from changedetectionio.processors import get_processor_submodule\n        processor_module = get_processor_submodule(processor_name, 'difference')\n\n        # Call the processor's get_asset() function\n        if processor_module and hasattr(processor_module, 'get_asset'):\n            result = processor_module.get_asset(\n                asset_name=asset_name,\n                watch=watch,\n                datastore=datastore,\n                request=request\n            )\n\n            if result is None:\n                from flask import abort\n                abort(404, description=f\"Asset '{asset_name}' not found\")\n\n            binary_data, content_type, cache_control = result\n\n            response = make_response(binary_data)\n            response.headers['Content-Type'] = content_type\n            if cache_control:\n                response.headers['Cache-Control'] = cache_control\n            return response\n        else:\n            logger.warning(f\"Processor {processor_name} does not implement get_asset()\")\n            from flask import abort\n            abort(404, description=f\"Processor '{processor_name}' does not support assets\")\n\n    return diff_blueprint\n"
  },
  {
    "path": "changedetectionio/blueprint/ui/edit.py",
    "content": "from copy import deepcopy\nimport os\nimport importlib.resources\nfrom flask import Blueprint, request, redirect, url_for, flash, render_template, abort\nfrom flask_babel import gettext\nfrom loguru import logger\nfrom jinja2 import Environment, FileSystemLoader\n\nfrom changedetectionio.store import ChangeDetectionStore\nfrom changedetectionio.auth_decorator import login_optionally_required\nfrom changedetectionio.time_handler import is_within_schedule\nfrom changedetectionio import worker_pool\n\ndef construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMetaData):\n    edit_blueprint = Blueprint('ui_edit', __name__, template_folder=\"../ui/templates\")\n    \n    def _watch_has_tag_options_set(watch):\n        \"\"\"This should be fixed better so that Tag is some proper Model, a tag is just a Watch also\"\"\"\n        for tag_uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():\n            if tag_uuid in watch.get('tags', []) and (tag.get('include_filters') or tag.get('subtractive_selectors')):\n                return True\n\n    @edit_blueprint.route(\"/edit/\", methods=['GET', 'POST'])\n    @login_optionally_required\n    # https://stackoverflow.com/questions/42984453/wtforms-populate-form-with-data-if-data-exists\n    # https://wtforms.readthedocs.io/en/3.0.x/forms/#wtforms.form.Form.populate_obj ?\n    def edit_page(uuid):\n        from changedetectionio import forms\n        from changedetectionio.browser_steps.browser_steps import browser_step_ui_config\n        from changedetectionio import processors\n        import importlib\n\n        if uuid == 'first':\n            uuid = list(datastore.data['watching'].keys()).pop()\n        # More for testing, possible to return the first/only\n        if not datastore.data['watching'].keys():\n            flash(gettext(\"No watches to edit\"), \"error\")\n            return redirect(url_for('watchlist.index'))\n\n        if not uuid in datastore.data['watching']:\n            flash(gettext(\"No watch with the UUID {} found.\").format(uuid), \"error\")\n            return redirect(url_for('watchlist.index'))\n\n        switch_processor = request.args.get('switch_processor')\n        if switch_processor:\n            for p in processors.available_processors():\n                if p[0] == switch_processor:\n                    datastore.data['watching'][uuid]['processor'] = switch_processor\n                    flash(gettext(\"Switched to mode - {}.\").format(p[1]))\n                    datastore.clear_watch_history(uuid)\n                    redirect(url_for('ui_edit.edit_page', uuid=uuid))\n\n        # be sure we update with a copy instead of accidently editing the live object by reference\n        default = None\n        while not default:\n            try:\n                default = deepcopy(datastore.data['watching'][uuid])\n            except RuntimeError as e:\n                # Dictionary changed\n                continue\n\n        # Defaults for proxy choice\n        if datastore.proxy_list is not None:  # When enabled\n            # @todo\n            # Radio needs '' not None, or incase that the chosen one no longer exists\n            if default['proxy'] is None or not any(default['proxy'] in tup for tup in datastore.proxy_list):\n                default['proxy'] = ''\n        # proxy_override set to the json/text list of the items\n\n        # Does it use some custom form? does one exist?\n        processor_name = datastore.data['watching'][uuid].get('processor', '')\n        processor_classes = next((tpl for tpl in processors.find_processors() if tpl[1] == processor_name), None)\n        if not processor_classes:\n            flash(gettext(\"Could not load '{}' processor, processor plugin might be missing. Please select a different processor.\").format(processor_name), 'error')\n            # Fall back to default processor so user can still edit and change processor\n            processor_classes = next((tpl for tpl in processors.find_processors() if tpl[1] == 'text_json_diff'), None)\n            if not processor_classes:\n                # If even text_json_diff is missing, something is very wrong\n                flash(gettext(\"Could not load '{}' processor, processor plugin might be missing.\").format(processor_name), 'error')\n                return redirect(url_for('watchlist.index'))\n\n        parent_module = processors.get_parent_module(processor_classes[0])\n\n        try:\n            # Get the parent of the \"processor.py\" go up one, get the form (kinda spaghetti but its reusing existing code)\n            forms_module = importlib.import_module(f\"{parent_module.__name__}.forms\")\n            # Access the 'processor_settings_form' class from the 'forms' module\n            form_class = getattr(forms_module, 'processor_settings_form')\n        except ModuleNotFoundError as e:\n            # .forms didnt exist\n            form_class = forms.processor_text_json_diff_form\n        except AttributeError as e:\n            # .forms exists but no useful form\n            form_class = forms.processor_text_json_diff_form\n\n        form = form_class(formdata=request.form if request.method == 'POST' else None,\n                          data=default,\n                          extra_notification_tokens=default.extra_notification_token_values(),\n                          default_system_settings=datastore.data['settings']\n                          )\n\n        # For the form widget tag UUID back to \"string name\" for the field\n        form.tags.datastore = datastore\n\n        # Used by some forms that need to dig deeper\n        form.datastore = datastore\n        form.watch = default\n\n        # Load processor-specific config from JSON file for GET requests\n        if request.method == 'GET' and processor_name:\n            try:\n                from changedetectionio.processors.base import difference_detection_processor\n                # Create a processor instance to access config methods\n                processor_instance = difference_detection_processor(datastore, uuid)\n                # Use processor name as filename so each processor keeps its own config\n                config_filename = f'{processor_name}.json'\n                processor_config = processor_instance.get_extra_watch_config(config_filename)\n\n                if processor_config:\n                    from wtforms.fields.form import FormField\n                    # Populate processor-config-* fields from JSON\n                    for config_key, config_value in processor_config.items():\n                        if not isinstance(config_value, dict):\n                            continue\n                        # Try exact API-named field first (e.g., processor_config_restock_diff)\n                        target_field = getattr(form, f'processor_config_{config_key}', None)\n                        # Fallback: find any FormField sub-form whose fields cover config_value keys\n                        if target_field is None:\n                            for form_field in form:\n                                if isinstance(form_field, FormField) and all(k in form_field.form._fields for k in config_value):\n                                    target_field = form_field\n                                    break\n                        if target_field is not None:\n                            for sub_key, sub_value in config_value.items():\n                                sub_field = target_field.form._fields.get(sub_key)\n                                if sub_field is not None:\n                                    sub_field.data = sub_value\n                                    logger.debug(f\"Loaded processor config from {config_filename}: {sub_key} = {sub_value}\")\n            except Exception as e:\n                logger.warning(f\"Failed to load processor config: {e}\")\n\n        for p in datastore.extra_browsers:\n            form.fetch_backend.choices.append(p)\n\n        form.fetch_backend.choices.append((\"system\", 'System settings default'))\n\n        # form.browser_steps[0] can be assumed that we 'goto url' first\n\n        if datastore.proxy_list is None:\n            # @todo - Couldn't get setattr() etc dynamic addition working, so remove it instead\n            del form.proxy\n        else:\n            form.proxy.choices = [('', 'Default')]\n            for p in datastore.proxy_list:\n                form.proxy.choices.append(tuple((p, datastore.proxy_list[p]['label'])))\n\n\n        if request.method == 'POST' and form.validate():\n\n            extra_update_obj = {\n                'consecutive_filter_failures': 0,\n                'last_error' : False\n            }\n\n            if request.args.get('unpause_on_save'):\n                extra_update_obj['paused'] = False\n\n            extra_update_obj['time_between_check'] = form.time_between_check.data\n\n            # Handle processor-config-* fields separately (save to JSON, not datastore)\n            # IMPORTANT: These must NOT be saved to url-watches.json, only to the processor-specific JSON file\n            processor_config_data = processors.extract_processor_config_from_form_data(form.data)\n            processors.save_processor_config(datastore, uuid, processor_config_data)\n\n            # Ignore text\n            form_ignore_text = form.ignore_text.data\n            datastore.data['watching'][uuid]['ignore_text'] = form_ignore_text\n\n            # Be sure proxy value is None\n            if datastore.proxy_list is not None and form.data['proxy'] == '':\n                extra_update_obj['proxy'] = None\n\n            # Unsetting all filter_text methods should make it go back to default\n            # This particularly affects tests running\n            if 'filter_text_added' in form.data and not form.data.get('filter_text_added') \\\n                    and 'filter_text_replaced' in form.data and not form.data.get('filter_text_replaced') \\\n                    and 'filter_text_removed' in form.data and not form.data.get('filter_text_removed'):\n                extra_update_obj['filter_text_added'] = True\n                extra_update_obj['filter_text_replaced'] = True\n                extra_update_obj['filter_text_removed'] = True\n\n            # Because wtforms doesn't support accessing other data in process_ , but we convert the CSV list of tags back to a list of UUIDs\n            tag_uuids = []\n            if form.data.get('tags'):\n                # Sometimes in testing this can be list, dont know why\n                if type(form.data.get('tags')) == list:\n                    extra_update_obj['tags'] = form.data.get('tags')\n                else:\n                    for t in form.data.get('tags').split(','):\n                        tag_uuids.append(datastore.add_tag(title=t))\n                    extra_update_obj['tags'] = tag_uuids\n\n            datastore.data['watching'][uuid].update(form.data)\n            datastore.data['watching'][uuid].update(extra_update_obj)\n\n            if not datastore.data['watching'][uuid].get('tags'):\n                # Force it to be a list, because form.data['tags'] will be string if nothing found\n                # And del(form.data['tags'] ) wont work either for some reason\n                datastore.data['watching'][uuid]['tags'] = []\n\n            # Recast it if need be to right data Watch handler\n            watch_class = processors.get_custom_watch_obj_for_processor(form.data.get('processor'))\n            datastore.data['watching'][uuid] = watch_class(datastore_path=datastore.datastore_path, __datastore=datastore.data, default=datastore.data['watching'][uuid])\n\n            # Save the watch immediately\n            datastore.data['watching'][uuid].commit()\n\n            flash(gettext(\"Updated watch - unpaused!\") if request.args.get('unpause_on_save') else gettext(\"Updated watch.\"))\n\n            # Cleanup any browsersteps session for this watch\n            try:\n                from changedetectionio.blueprint.browser_steps import cleanup_session_for_watch\n                cleanup_session_for_watch(uuid)\n            except Exception as e:\n                logger.debug(f\"Error cleaning up browsersteps session: {e}\")\n\n            # Do not queue on edit if its not within the time range\n\n            # @todo maybe it should never queue anyway on edit...\n            is_in_schedule = True\n            watch = datastore.data['watching'].get(uuid)\n\n            if watch.get('time_between_check_use_default'):\n                time_schedule_limit = datastore.data['settings']['requests'].get('time_schedule_limit', {})\n            else:\n                time_schedule_limit = watch.get('time_schedule_limit')\n\n            tz_name = time_schedule_limit.get('timezone')\n            if not tz_name:\n                tz_name = datastore.data['settings']['application'].get('scheduler_timezone_default', os.getenv('TZ', 'UTC').strip())\n\n            if time_schedule_limit and time_schedule_limit.get('enabled'):\n                try:\n                    is_in_schedule = is_within_schedule(time_schedule_limit=time_schedule_limit,\n                                                      default_tz=tz_name\n                                                      )\n                except Exception as e:\n                    logger.error(\n                        f\"{uuid} - Recheck scheduler, error handling timezone, check skipped - TZ name '{tz_name}' - {str(e)}\")\n                    return False\n\n            #############################\n            if not datastore.data['watching'][uuid].get('paused') and is_in_schedule:\n                # Queue the watch for immediate recheck, with a higher priority\n                worker_pool.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))\n\n            # Diff page [edit] link should go back to diff page\n            if request.args.get(\"next\") and request.args.get(\"next\") == 'diff':\n                return redirect(url_for('ui.ui_diff.diff_history_page', uuid=uuid))\n\n            return redirect(url_for('watchlist.index', tag=request.args.get(\"tag\",'')))\n\n        else:\n            if request.method == 'POST' and not form.validate():\n                flash(gettext(\"An error occurred, please see below.\"), \"error\")\n\n            # JQ is difficult to install on windows and must be manually added (outside requirements.txt)\n            jq_support = True\n            try:\n                import jq\n            except ModuleNotFoundError:\n                jq_support = False\n\n            watch = datastore.data['watching'].get(uuid)\n\n            from zoneinfo import available_timezones\n\n            # Import the global plugin system\n            from changedetectionio.pluggy_interface import collect_ui_edit_stats_extras, get_fetcher_capabilities\n\n            # Get fetcher capabilities instead of hardcoded logic\n            capabilities = get_fetcher_capabilities(watch, datastore)\n\n            # Add processor capabilities from module\n            capabilities['supports_visual_selector'] = getattr(parent_module, 'supports_visual_selector', False)\n            capabilities['supports_text_filters_and_triggers'] = getattr(parent_module, 'supports_text_filters_and_triggers', False)\n            capabilities['supports_text_filters_and_triggers_elements'] = getattr(parent_module, 'supports_text_filters_and_triggers_elements', False)\n            capabilities['supports_request_type'] = getattr(parent_module, 'supports_request_type', False)\n\n            app_rss_token = datastore.data['settings']['application'].get('rss_access_token'),\n\n            c = [f\"processor-{watch.get('processor')}\"]\n            if worker_pool.is_watch_running(uuid):\n                c.append('checking-now')\n\n            template_args = {\n                'available_processors': processors.available_processors(),\n                'available_timezones': sorted(available_timezones()),\n                'browser_steps_config': browser_step_ui_config,\n                'emailprefix': os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),\n                'extra_classes': ' '.join(c),\n                'extra_notification_token_placeholder_info': datastore.get_unique_notification_token_placeholders_available(),\n                'extra_processor_config': form.extra_tab_content(),\n                'extra_title': f\" - Edit - {watch.label}\",\n                'form': form,\n                'has_default_notification_urls': True if len(datastore.data['settings']['application']['notification_urls']) else False,\n                'has_extra_headers_file': len(datastore.get_all_headers_in_textfile_for_watch(uuid=uuid)) > 0,\n                'has_special_tag_options': _watch_has_tag_options_set(watch=watch),\n                'jq_support': jq_support,\n                'playwright_enabled': os.getenv('PLAYWRIGHT_DRIVER_URL', False),\n                'app_rss_token': app_rss_token,\n                'rss_uuid_feed' : {\n                    'label': watch.label,\n                    'url': url_for('rss.rss_single_watch', uuid=watch['uuid'], token=app_rss_token)\n                },\n                'settings_application': datastore.data['settings']['application'],\n                'ui_edit_stats_extras': collect_ui_edit_stats_extras(watch),\n                'visual_selector_data_ready': datastore.visualselector_data_is_ready(watch_uuid=uuid),\n                'timezone_default_config': datastore.data['settings']['application'].get('scheduler_timezone_default'),\n                'using_global_webdriver_wait': not default['webdriver_delay'],\n                'uuid': uuid,\n                'watch': watch,\n                'capabilities': capabilities\n            }\n\n            included_content = None\n            if form.extra_form_content():\n                # So that the extra panels can access _helpers.html etc, we set the environment to load from templates/\n                # And then render the code from the module\n                templates_dir = str(importlib.resources.files(\"changedetectionio\").joinpath('templates'))\n                env = Environment(loader=FileSystemLoader(templates_dir))\n                template = env.from_string(form.extra_form_content())\n                included_content = template.render(**template_args)\n\n            output = render_template(\"edit.html\",\n                                     extra_tab_content=form.extra_tab_content() if form.extra_tab_content() else None,\n                                     extra_form_content=included_content,\n                                     **template_args\n                                     )\n\n        return output\n\n    @edit_blueprint.route(\"/edit//get-html\", methods=['GET'])\n    @login_optionally_required\n    def watch_get_latest_html(uuid):\n        from io import BytesIO\n        from flask import send_file\n        import brotli\n\n        if uuid == 'first':\n            uuid = list(datastore.data['watching'].keys()).pop()\n        watch = datastore.data['watching'].get(uuid)\n        if watch and watch.history.keys() and os.path.isdir(watch.data_dir):\n            latest_filename = list(watch.history.keys())[-1]\n            html_fname = os.path.join(watch.data_dir, f\"{latest_filename}.html.br\")\n            with open(html_fname, 'rb') as f:\n                if html_fname.endswith('.br'):\n                    # Read and decompress the Brotli file\n                    decompressed_data = brotli.decompress(f.read())\n                else:\n                    decompressed_data = f.read()\n\n            buffer = BytesIO(decompressed_data)\n\n            return send_file(buffer, as_attachment=True, download_name=f\"{latest_filename}.html\", mimetype='text/html')\n\n        # Return a 500 error\n        abort(500)\n\n    @edit_blueprint.route(\"/edit//get-data-package\", methods=['GET'])\n    @login_optionally_required\n    def watch_get_data_package(uuid):\n        \"\"\"Download all data for a single watch as a zip file\"\"\"\n        from io import BytesIO\n        from flask import send_file\n        import zipfile\n        from pathlib import Path\n        import datetime\n\n        watch = datastore.data['watching'].get(uuid)\n        if not watch:\n            abort(404)\n\n        # Create zip in memory\n        memory_file = BytesIO()\n\n        with zipfile.ZipFile(memory_file, 'w',\n                           compression=zipfile.ZIP_DEFLATED,\n                           compresslevel=8) as zipObj:\n\n            # Add the watch's JSON file if it exists\n            watch_json_path = os.path.join(watch.data_dir, 'watch.json')\n            if os.path.isfile(watch_json_path):\n                zipObj.write(watch_json_path,\n                           arcname=os.path.join(uuid, 'watch.json'),\n                           compress_type=zipfile.ZIP_DEFLATED,\n                           compresslevel=8)\n\n            # Add all files in the watch data directory\n            if os.path.isdir(watch.data_dir):\n                for f in Path(watch.data_dir).glob('*'):\n                    if f.is_file() and f.name != 'watch.json':  # Skip watch.json since we already added it\n                        zipObj.write(f,\n                                   arcname=os.path.join(uuid, f.name),\n                                   compress_type=zipfile.ZIP_DEFLATED,\n                                   compresslevel=8)\n\n        # Seek to beginning of file\n        memory_file.seek(0)\n\n        # Generate filename with timestamp\n        timestamp = datetime.datetime.now().strftime(\"%Y%m%d%H%M%S\")\n        filename = f\"watch-data-{uuid[:8]}-{timestamp}.zip\"\n\n        return send_file(memory_file,\n                        as_attachment=True,\n                        download_name=filename,\n                        mimetype='application/zip')\n\n    # Ajax callback\n    @edit_blueprint.route(\"/edit//preview-rendered\", methods=['POST'])\n    @login_optionally_required\n    def watch_get_preview_rendered(uuid):\n        '''For when viewing the \"preview\" of the rendered text from inside of Edit'''\n        from flask import jsonify\n\n        if uuid == 'first':\n            uuid = list(datastore.data['watching'].keys()).pop()\n        from changedetectionio.processors.text_json_diff import prepare_filter_prevew\n        result = prepare_filter_prevew(watch_uuid=uuid, form_data=request.form, datastore=datastore)\n        return jsonify(result)\n\n    @edit_blueprint.route(\"/highlight_submit_ignore_url\", methods=['POST'])\n    @login_optionally_required\n    def highlight_submit_ignore_url():\n        import re\n        mode = request.form.get('mode')\n        selection = request.form.get('selection')\n\n        uuid = request.args.get('uuid','')\n        if datastore.data[\"watching\"].get(uuid):\n            if mode == 'exact':\n                for l in selection.splitlines():\n                    datastore.data[\"watching\"][uuid]['ignore_text'].append(l.strip())\n            elif mode == 'digit-regex':\n                for l in selection.splitlines():\n                    # Replace any series of numbers with a regex\n                    s = re.escape(l.strip())\n                    s = re.sub(r'[0-9]+', r'\\\\d+', s)\n                    datastore.data[\"watching\"][uuid]['ignore_text'].append('/' + s + '/')\n\n            # Save the updated ignore_text\n            datastore.data[\"watching\"][uuid].commit()\n\n        return f\"Click to preview\"\n    \n    return edit_blueprint"
  },
  {
    "path": "changedetectionio/blueprint/ui/notification.py",
    "content": "from flask import Blueprint, request, make_response\nimport random\nfrom loguru import logger\n\nfrom changedetectionio.store import ChangeDetectionStore\nfrom changedetectionio.auth_decorator import login_optionally_required\n\ndef construct_blueprint(datastore: ChangeDetectionStore):\n    notification_blueprint = Blueprint('ui_notification', __name__, template_folder=\"../ui/templates\")\n    \n    # AJAX endpoint for sending a test\n    @notification_blueprint.route(\"/notification/send-test/\", methods=['POST'])\n    @notification_blueprint.route(\"/notification/send-test\", methods=['POST'])\n    @notification_blueprint.route(\"/notification/send-test/\", methods=['POST'])\n    @login_optionally_required\n    def ajax_callback_send_notification_test(watch_uuid=None):\n        from changedetectionio.notification_service import NotificationContextData, set_basic_notification_vars\n        # Watch_uuid could be unset in the case it`s used in tag editor, global settings\n        import apprise\n        from changedetectionio.notification.handler import process_notification\n        from changedetectionio.notification.apprise_plugin.assets import apprise_asset\n        from changedetectionio.jinja2_custom import render as jinja_render\n\n        from changedetectionio.notification.apprise_plugin.custom_handlers import apprise_http_custom_handler\n\n        apobj = apprise.Apprise(asset=apprise_asset)\n\n        is_global_settings_form = request.args.get('mode', '') == 'global-settings'\n        is_group_settings_form = request.args.get('mode', '') == 'group-settings'\n\n        # Use an existing random one on the global/main settings form\n        if not watch_uuid and (is_global_settings_form or is_group_settings_form) \\\n                and datastore.data.get('watching'):\n            logger.debug(f\"Send test notification - Choosing random Watch {watch_uuid}\")\n            watch_uuid = random.choice(list(datastore.data['watching'].keys()))\n\n        if not watch_uuid:\n            return make_response(\"Error: You must have atleast one watch configured for 'test notification' to work\", 400)\n\n        watch = datastore.data['watching'].get(watch_uuid)\n        notification_urls = request.form.get('notification_urls','').strip().splitlines()\n\n        if not notification_urls:\n            logger.debug(\"Test notification - Trying by group/tag in the edit form if available\")\n            # On an edit page, we should also fire off to the tags if they have notifications\n            if request.form.get('tags') and request.form['tags'].strip():\n                for k in request.form['tags'].split(','):\n                    tag = datastore.tag_exists_by_name(k.strip())\n                    notification_urls = tag.get('notifications_urls') if tag and tag.get('notifications_urls') else None\n\n        if not notification_urls and not is_global_settings_form and not is_group_settings_form:\n            # In the global settings, use only what is typed currently in the text box\n            logger.debug(\"Test notification - Trying by global system settings notifications\")\n            if datastore.data['settings']['application'].get('notification_urls'):\n                notification_urls = datastore.data['settings']['application']['notification_urls']\n\n        if not notification_urls:\n            return 'Error: No Notification URLs set/found'\n\n        for n_url in notification_urls:\n            # We are ONLY validating the apprise:// part here, convert all tags to something so as not to break apprise URLs\n            generic_notification_context_data = NotificationContextData()\n            generic_notification_context_data.set_random_for_validation()\n            n_url = jinja_render(template_str=n_url, **generic_notification_context_data).strip()\n            if len(n_url.strip()):\n                if not apobj.add(n_url):\n                    return f'Error:  {n_url} is not a valid AppRise URL.'\n\n        try:\n            # use the same as when it is triggered, but then override it with the form test values\n            n_object = NotificationContextData({\n                'watch_url': request.form.get('window_url', \"https://changedetection.io\"),\n                'notification_urls': notification_urls\n            })\n\n            # Only use if present, if not set in n_object it should use the default system value\n            if 'notification_format' in request.form and request.form['notification_format'].strip():\n                n_object['notification_format'] = request.form.get('notification_format', '').strip()\n            else:\n                n_object['notification_format'] = datastore.data['settings']['application'].get('notification_format')\n\n            if 'notification_title' in request.form and request.form['notification_title'].strip():\n                n_object['notification_title'] = request.form.get('notification_title', '').strip()\n            elif datastore.data['settings']['application'].get('notification_title'):\n                n_object['notification_title'] = datastore.data['settings']['application'].get('notification_title')\n            else:\n                n_object['notification_title'] = \"Test title\"\n\n            if 'notification_body' in request.form and request.form['notification_body'].strip():\n                n_object['notification_body'] = request.form.get('notification_body', '').strip()\n            elif datastore.data['settings']['application'].get('notification_body'):\n                n_object['notification_body'] = datastore.data['settings']['application'].get('notification_body')\n            else:\n                n_object['notification_body'] = \"Test body\"\n\n            n_object['as_async'] = False\n\n            #  Same like in notification service, should be refactored\n            dates = list(watch.history.keys())\n            trigger_text = ''\n            snapshot_contents = ''\n\n            # Could be called as a 'test notification' with only 1 snapshot available\n            prev_snapshot = \"Example text: example test\\nExample text: change detection is cool\\nExample text: some more examples\\n\"\n            current_snapshot = \"Example text: example test\\nExample text: change detection is fantastic\\nExample text: even more examples\\nExample text: a lot more examples\"\n\n            if len(dates) > 1:\n                prev_snapshot = watch.get_history_snapshot(timestamp=dates[-2])\n                current_snapshot = watch.get_history_snapshot(timestamp=dates[-1])\n\n            n_object.update(set_basic_notification_vars(current_snapshot=current_snapshot,\n                                                        prev_snapshot=prev_snapshot,\n                                                        watch=watch,\n                                                        triggered_text=trigger_text,\n                                                        timestamp_changed=dates[-1] if dates else None))\n\n\n            sent_obj = process_notification(n_object, datastore)\n\n        except Exception as e:\n            logger.error(e)\n            e_str = str(e)\n            # Remove this text which is not important and floods the container\n            e_str = e_str.replace(\n                \"DEBUG - .CustomNotifyPluginWrapper'>\",\n                '')\n\n            return make_response(e_str, 400)\n\n        return 'OK - Sent test notifications'\n\n    return notification_blueprint"
  },
  {
    "path": "changedetectionio/blueprint/ui/preview.py",
    "content": "from flask import Blueprint, request, url_for, flash, render_template, redirect\nfrom flask_babel import gettext\nimport time\nfrom loguru import logger\n\nfrom changedetectionio.store import ChangeDetectionStore\nfrom changedetectionio.auth_decorator import login_optionally_required\nfrom changedetectionio import html_tools\n\ndef construct_blueprint(datastore: ChangeDetectionStore):\n    preview_blueprint = Blueprint('ui_preview', __name__, template_folder=\"../ui/templates\")\n\n\n    @preview_blueprint.route(\"/preview/\", methods=['GET', 'POST'])\n    @login_optionally_required\n    def preview_page(uuid):\n        \"\"\"\n        Render the preview page for a watch.\n\n        This route is processor-aware: it delegates rendering to the processor's\n        preview.py module, allowing different processor types to provide\n        custom visualizations:\n        - text_json_diff: Text preview with syntax highlighting\n        - image_ssim_diff: Image preview with proper rendering\n        - restock_diff: Could show latest price/stock data\n\n        Each processor implements processors/{type}/preview.py::render()\n        If a processor doesn't have a preview module, falls back to default text preview.\n        \"\"\"\n\n        if uuid == 'first':\n            uuid = list(datastore.data['watching'].keys()).pop()\n        try:\n            watch = datastore.data['watching'][uuid]\n        except KeyError:\n            flash(gettext(\"No history found for the specified link, bad link?\"), \"error\")\n            return redirect(url_for('watchlist.index'))\n\n        # Get the processor type for this watch\n        processor_name = watch.get('processor', 'text_json_diff')\n\n        # Try to get the processor's preview module (works for both built-in and plugin processors)\n        from changedetectionio.processors import get_processor_submodule\n        processor_module = get_processor_submodule(processor_name, 'preview')\n\n        # Call the processor's render() function\n        if processor_module and hasattr(processor_module, 'render'):\n            return processor_module.render(\n                watch=watch,\n                datastore=datastore,\n                request=request,\n                url_for=url_for,\n                render_template=render_template,\n                flash=flash,\n                redirect=redirect\n            )\n\n        # Fallback: if processor doesn't have preview module, use default text preview\n        content = []\n        versions = []\n        timestamp = None\n\n        extra_stylesheets = [url_for('static_content', group='styles', filename='diff.css')]\n        is_html_webdriver = watch.fetcher_supports_screenshots\n\n        triggered_line_numbers = []\n        ignored_line_numbers = []\n        blocked_line_numbers = []\n\n        if datastore.data['watching'][uuid].history_n == 0 and (watch.get_error_text() or watch.get_error_snapshot()):\n            flash(gettext(\"Preview unavailable - No fetch/check completed or triggers not reached\"), \"error\")\n        else:\n            # So prepare the latest preview or not\n            preferred_version = request.values.get('version') if request.method == 'POST' else request.args.get('version')\n\n\n            versions = list(watch.history.keys())\n            timestamp = versions[-1]\n            if preferred_version and preferred_version in versions:\n                timestamp = preferred_version\n\n            try:\n                versions = list(watch.history.keys())\n                content = watch.get_history_snapshot(timestamp=timestamp)\n\n                triggered_line_numbers = html_tools.strip_ignore_text(content=content,\n                                                                      wordlist=watch.get('trigger_text'),\n                                                                      mode='line numbers'\n                                                                      )\n                ignored_line_numbers = html_tools.strip_ignore_text(content=content,\n                                                                      wordlist=watch.get('ignore_text'),\n                                                                      mode='line numbers'\n                                                                      )\n                blocked_line_numbers = html_tools.strip_ignore_text(content=content,\n                                                                      wordlist=watch.get(\"text_should_not_be_present\"),\n                                                                      mode='line numbers'\n                                                                      )\n            except Exception as e:\n                content.append({'line': f\"File doesnt exist or unable to read timestamp {timestamp}\", 'classes': ''})\n\n        from changedetectionio.pluggy_interface import get_fetcher_capabilities\n        capabilities = get_fetcher_capabilities(watch, datastore)\n\n        output = render_template(\"preview.html\",\n                                 capabilities=capabilities,\n                                 content=content,\n                                 current_diff_url=watch['url'],\n                                 current_version=timestamp,\n                                 extra_stylesheets=extra_stylesheets,\n                                 extra_title=f\" - Diff - {watch.label} @ {timestamp}\",\n                                 highlight_ignored_line_numbers=ignored_line_numbers,\n                                 highlight_triggered_line_numbers=triggered_line_numbers,\n                                 highlight_blocked_line_numbers=blocked_line_numbers,\n                                 history_n=watch.history_n,\n                                 is_html_webdriver=is_html_webdriver,\n                                 last_error=watch['last_error'],\n                                 last_error_screenshot=watch.get_error_snapshot(),\n                                 last_error_text=watch.get_error_text(),\n                                 screenshot=watch.get_screenshot(),\n                                 uuid=uuid,\n                                 versions=versions,\n                                 watch=watch,\n                                 )\n\n        return output\n\n    @preview_blueprint.route(\"/preview//processor-asset/\", methods=['GET'])\n    @login_optionally_required\n    def processor_asset(uuid, asset_name):\n        \"\"\"\n        Serve processor-specific binary assets for preview (images, files, etc.).\n\n        This route is processor-aware: it delegates to the processor's\n        preview.py module, allowing different processor types to serve\n        custom assets without embedding them as base64 in templates.\n\n        This solves memory issues with large binary data by streaming them\n        as separate HTTP responses instead of embedding in the HTML template.\n\n        Each processor implements processors/{type}/preview.py::get_asset()\n        which returns (binary_data, content_type, cache_control_header).\n\n        Example URLs:\n        - /preview/{uuid}/processor-asset/screenshot?version=123456789\n        \"\"\"\n        from flask import make_response\n\n        if uuid == 'first':\n            uuid = list(datastore.data['watching'].keys()).pop()\n        try:\n            watch = datastore.data['watching'][uuid]\n        except KeyError:\n            flash(gettext(\"No history found for the specified link, bad link?\"), \"error\")\n            return redirect(url_for('watchlist.index'))\n\n        # Get the processor type for this watch\n        processor_name = watch.get('processor', 'text_json_diff')\n\n        # Try to get the processor's preview module (works for both built-in and plugin processors)\n        from changedetectionio.processors import get_processor_submodule\n        processor_module = get_processor_submodule(processor_name, 'preview')\n\n        # Call the processor's get_asset() function\n        if processor_module and hasattr(processor_module, 'get_asset'):\n            result = processor_module.get_asset(\n                asset_name=asset_name,\n                watch=watch,\n                datastore=datastore,\n                request=request\n            )\n\n            if result is None:\n                from flask import abort\n                abort(404, description=f\"Asset '{asset_name}' not found\")\n\n            binary_data, content_type, cache_control = result\n\n            response = make_response(binary_data)\n            response.headers['Content-Type'] = content_type\n            if cache_control:\n                response.headers['Cache-Control'] = cache_control\n            return response\n        else:\n            logger.warning(f\"Processor {processor_name} does not implement get_asset()\")\n            from flask import abort\n            abort(404, description=f\"Processor '{processor_name}' does not support assets\")\n\n    return preview_blueprint\n"
  },
  {
    "path": "changedetectionio/blueprint/ui/templates/clear_all_history.html",
    "content": "{% extends 'base.html' %} {% block content %}\n
\n  
\n    \n      \n      
\n        
\n          {{ _('This will remove version history (snapshots) for ALL watches, but keep your list of URLs!') }} 
\n          {{ _('You may like to use the') }} {{ _('BACKUP') }} {{ _('link first.') }}
\n        
\n        
\n        
\n          \n          \n          {{ _('Type in the word') }} {{ _('clear') }} {{ _('to confirm that you understand.') }}\n        
\n        
\n        
\n          \n        
\n        
\n        
\n          {{ _('Cancel') }}\n        
\n      
\n    \n  
\n
\n\n{% endblock %}\n"
  },
  {
    "path": "changedetectionio/blueprint/ui/templates/diff-offscreen-options.html",
    "content": "\n\n"
  },
  {
    "path": "changedetectionio/blueprint/ui/templates/diff.html",
    "content": "{% extends 'base.html' %}\n{% from '_helpers.html' import render_field, render_checkbox_field, render_button %}\n{% block content %}\n\n\n\n\n\n\n\n\n
\n    
\n        
\n            {% if versions|length >= 1 %}\n                \n                \n                \n                \n                \n                \n                \n                \n                {##}\n            {% endif %}\n        
\n        
\n            \n                \n            \n            \n                \n            \n            \n                \n            \n            \n                \n            \n            \n                \n            \n            \n                \n            \n            \n                \n            \n        
\n        {%- if versions|length >= 2 -%}\n            
\n                {{ _('Keyboard:') }} \n                 ← {{ _('Previous') }}\n                   → {{ _('Next') }}\n            
\n        {%- endif -%}\n    
\n
\n\n\n\n\n
\n    \n
\n\n
\n    
\n        
{{watch_a.error_text_ctime|format_seconds_ago}} {{ _('seconds ago.') }}
\n        
\n            {{ last_error_text }}\n        
\n    
\n\n    
\n        
{{watch_a.snapshot_error_screenshot_ctime|format_seconds_ago}} {{ _('seconds ago') }}
\n        \"{{\n    
\n\n    
\n    {%- if (content | default('')).split('\\n') | length > 100 -%}\n        
\n            {%- for cell in diff_cell_grid -%}\n            
\n            {%- endfor -%}\n        
\n    {%- endif -%}\n        {%- if password_enabled_and_share_is_off -%}\n            
{{ _('Pro-tip: You can enable') }} {{ _('\"share access when password is enabled\"') }} {{ _('from settings.') }}\n            
\n        {%- endif -%}\n        
\n            
{{ from_version|format_timestamp_timeago }}\n                {%- if note -%}{{ note }}{%- endif -%}\n                {{ _('Goto single snapshot') }}\n            
\n        
\n        
{{ content| diff_unescape_difference_spans }}
\n    
\n        {{ _('Tip:') }} {{ _('Highlight text to share or add to ignore lists.') }}\n    
\n    
\n\n    
\n         
\n             {{ _('For now, Differences are performed on text, not graphically, only the latest screenshot is available.') }}\n         
\n         {% if is_html_webdriver %}\n           {% if screenshot %}\n            
{{watch_a.snapshot_screenshot_ctime|format_timestamp_timeago}}
\n            \"{{\n           {% else %}\n              {{ _('No screenshot available just yet! Try rechecking the page.') }}\n           {% endif %}\n         {% else %}\n           {{ _('Screenshot requires Playwright/WebDriver enabled') }}\n         {% endif %}\n     
\n\n\n\n\n\n\n\n{% endblock %}\n"
  },
  {
    "path": "changedetectionio/blueprint/ui/templates/edit.html",
    "content": "{% extends 'base.html' %}\n{% block content %}\n{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form, playwright_warning, only_playwright_type_watches_warning, highlight_trigger_ignored_explainer, render_conditions_fieldlist_of_formfields_as_table, render_ternary_field %}\n{% from '_common_fields.html' import render_common_settings_form %}\n\n\n\n\n\n\n\n\n\n\n\n\n\n{% if playwright_enabled %}\n\n{% endif %}\n\n{% set has_tag_filters_extra=\"WARNING: Watch has tag/groups set with special filters\\n\" if has_special_tag_options else '' %}\n\n\n
\n\n    
\n        \n    
\n\n    
\n        
\n             \n\n            
\n                
\n                    
\n                        {{ render_field(form.url, placeholder=\"https://...\", required=true, class=\"m-d\") }}\n                        
{{ _('Some sites use JavaScript to create the content, for this you should') }} {{ _('use the Chrome/WebDriver Fetcher') }}
\n                        
{{ _('Variables are supported in the URL') }} ({{ _('help and examples here') }}).
\n                    
\n                    
\n                        {{ render_field(form.tags) }}\n                        {{ _('Organisational tag/group name used in the main listing page') }}\n                    
\n                    
\n                        {{ render_field(form.processor) }}\n                    
\n                    
\n                        {{ render_field(form.title, class=\"m-d\", placeholder=watch.label) }}\n                        {{ _('Automatically uses the page title if found, you can also use your own title/description here') }}\n                    
\n                    
\n\n                        {{ render_checkbox_field(form.time_between_check_use_default, class=\"use-default-timecheck\") }}\n                        
\n                        
\n                            {{ render_field(form.time_between_check, class=\"time-check-widget\") }}\n\n                            \n                             {{ _('The interval/amount of time between each check.') }}\n                            \n                        
\n                        
\n                            \n                            
\n                                {{ render_time_schedule_form(form, available_timezones, timezone_default_config) }}\n                            
\n                        
\n
\n              
\n\n                    
\n                        {{ render_checkbox_field(form.filter_failure_notification_send) }}\n                        \n                         {{ _('Sends a notification when the filter can no longer be seen on the page, good for knowing when the page changed and your filter will not work anymore.') }}\n                        \n                    
\n                    
\n                        {{ render_field(form.history_snapshot_max_length, class=\"history_snapshot_max_length\") }}\n                        {{ _('Limit collection of history snapshots for each watch to this number of history items.') }}\n                        
\n                        {{ _('Set to empty to use system settings default') }}\n                        \n                    
\n                    
\n                        {{ render_ternary_field(form.use_page_title_in_list) }}\n                    
\n                
\n            
\n\n            {% if capabilities.supports_request_type %}\n            
\n                    
\n                        {{ render_field(form.fetch_backend, class=\"fetch-backend\") }}\n                        \n                            
{{ _('Use the') }} {{ _('Basic') }} {{ _('method (default) where your watched site doesn\\'t need Javascript to render.') }}
\n                            
{{ _('The') }} {{ _('Chrome/Javascript') }} {{ _('method requires a network connection to a running WebDriver+Chrome server, set by the ENV var \\'WEBDRIVER_URL\\'.') }} 
\n                            {{ _('Tip:') }} {{ _('Connect using Bright Data and Oxylabs Proxies, find out more here.') }}\n                        \n                    
\n                {% if form.proxy %}\n                    
\n                          
{{ form.proxy.label }} {{ _('Check/Scan all') }}
\n                          
{{ form.proxy(class=\"fetch-backend-proxy\") }}
\n                        \n                        {{ _('Choose a proxy for this watch') }}\n                        \n                    
\n                {% endif %}\n\n                \n                
\n                    
\n                        {{ render_field(form.webdriver_delay) }}\n                        
\n                            {{ _('If you\\'re having trouble waiting for the page to be fully rendered (text missing etc), try increasing the \\'wait\\' time here.') }}\n                            
\n                            {{ _('This will wait') }} n {{ _('seconds before extracting the text.') }}\n                            {% if using_global_webdriver_wait %}\n                            
{{ _('Using the current global default settings') }}\n                            {% endif %}\n                        
\n                    
\n                    
\n                        {{ _('Show advanced options') }}\n                    
\n                    
\n                        {{ render_field(form.webdriver_js_execute_code) }}\n                        
\n                            {{ _('Run this code before performing change detection, handy for filling in fields and other actions') }} {{ _('More help and examples here') }}\n                        
\n                    
\n                
\n                \n                
\n                    
\n                        {{ _('Show advanced options') }}\n                    
\n                    
\n                        
\n                            {{ render_field(form.method) }}\n                        
\n                        
\n                                            {{ render_field(form.body, rows=7, placeholder=\"Example\n{\n   \\\"name\\\":\\\"John\\\",\n   \\\"age\\\":30,\n   \\\"car\\\":null,\n   \\\"year\\\":{% now 'Europe/Berlin', '%Y' %}\n}\") }}\n                        
\n                        
{{ _('Variables are supported in the request body') }} ({{ _('help and examples here') }}).
\n                    
\n                
\n            \n                
\n                    {{ render_field(form.headers, rows=7, placeholder=\"Example\nCookie: foobar\nUser-Agent: wonderbra 1.0\nMath: {{ 1 + 1 }}\") }}\n                        
{{ _('Variables are supported in the request header values') }} ({{ _('help and examples here') }}).
\n                        
\n                            {% if has_extra_headers_file %}\n                                {{ _('Alert! Extra headers file found and will be added to this watch!') }}\n                            {% else %}\n                                {{ _('Headers can be also read from a file in your data-directory') }} {{ _('Read more here') }}\n                            {% endif %}\n                            
\n                            ({{ _('Not supported by Selenium browser') }})\n                        
\n                    
\n            
\n                    
\n                    {{ render_checkbox_field(form.ignore_status_codes) }}\n                    
\n            
\n            
\n            {% endif %}\n\n            
\n            {% if capabilities.supports_browser_steps %}\n               {% if true %}\n                \"New\n                
\n                    
\n                        \n\n                        \n                        
\n                             \n                        
\n\n                        
{{ _('Please wait, first browser step can take a little time to load..') }}
\n                        
\n\n                            
\n                                
\n                                    \n                                        \n                                            
{{ _('Click here to Start') }}
\n                                            
\n                                            {{ _('Please allow 10-15 seconds for the browser to connect.') }}
\n                                        \n                                        
\n                                    \n                                    \n                                    \n                                
\n                            
\n                            
\n                                {{ _('Press \"Play\" to start.') }}  (?) \n                                {{ render_field(form.browser_steps) }}\n                            
\n                        
\n                    
\n                
\n                {% else %}\n                    {{ _('Visual Selector data is not ready, watch needs to be checked atleast once.') }}\n                {% endif %}\n            {% else %}\n                
\n                    {{ _('Sorry, this functionality only works with fetchers that support interactive Javascript (so far only Playwright based fetchers)') }}
\n                    {{ _('You need to') }} {{ _('Set the fetch method') }} {{ _('to one that supports interactive Javascript.') }}\n                
\n            {% endif %}\n            
\n\n\n            
\n                
\n                    
\n                      {{ render_ternary_field(form.notification_muted, BooleanField=true) }}\n                    
\n                    {% if capabilities.supports_screenshots %}\n                    
\n                      {{ render_checkbox_field(form.notification_screenshot) }}\n                        \n                            {{ _('Use with caution!') }} {{ _('This will easily fill up your email storage quota or flood other storages.') }}\n                        \n                    
\n                    {% endif %}\n                    
\n                        {% if has_default_notification_urls %}\n                        
\n                            \"{{\n                            {{ _('There are') }} {{ _('system-wide notification URLs enabled') }}, {{ _('this form will override notification settings for this watch only') }} ‐ {{ _('an empty Notification URL list here will still send notifications.') }}\n                        
\n                        {% endif %}\n                        {{ _('Use system defaults') }}\n                        {{ render_common_settings_form(form, emailprefix, settings_application, extra_notification_token_placeholder_info) }}\n                    
\n                
\n            
\n\n            {% if capabilities.supports_text_filters_and_triggers %}\n            
\n                    \n                
\n                    {{ render_field(form.conditions_match_logic) }}\n                    {{ render_conditions_fieldlist_of_formfields_as_table(form.conditions) }}\n                    
\n\n                        
{{ _('Use the verify (✓) button to test if a condition passes against the current snapshot.') }}
\n                       {{ _('Read a quick tutorial about') }} {{ _('using conditional web page changes here') }}.
\n                    
\n                
\n            
\n            
\n                {{ _('Activate preview') }}\n              
\n              
\n\n{% if capabilities.supports_text_filters_and_triggers_elements %}\n                        
\n                            {{ _('Pro-tips:') }}
\n                            
    \n                                
  • \n                                    {{ _('Use the preview page to see your filters and triggers highlighted.') }}\n                                
    • \n                                
  • \n                                    {{ _('Some sites use JavaScript to create the content, for this you should') }} {{ _('use the Chrome/WebDriver Fetcher') }}\n                                
    • \n                            
\n                    
\n{% include \"edit/include_subtract.html\" %}\n{% endif %}\n                
\n                
\n                    
{{ _('Text filtering') }}
\n                        {{ _('Limit trigger/ignore/block/extract to;') }}
\n                        {{ render_checkbox_field(form.filter_text_added) }}\n                        {{ render_checkbox_field(form.filter_text_replaced) }}\n                        {{ render_checkbox_field(form.filter_text_removed) }}\n                    {{ _('Note: Depending on the length and similarity of the text on each line, the algorithm may consider an') }} {{ _('addition') }} {{ _('instead of') }} {{ _('replacement') }} {{ _('for example.') }}
\n                     {{ _('So it\\'s always better to select') }} {{ _('Added') }}+{{ _('Replaced') }} {{ _('when you\\'re interested in new content.') }}
\n                     {{ _('When content is merely moved in a list, it will also trigger an') }} {{ _('addition') }}, {{ _('consider enabling') }} {{ _('Only trigger when unique lines appear') }}\n                
\n                
\n                    {{ render_checkbox_field(form.check_unique_lines) }}\n                    {{ _('Good for websites that just move the content around, and you want to know when NEW content is added, compares new lines against all history for this watch.') }}\n                
\n                
\n                    {{ render_checkbox_field(form.remove_duplicate_lines) }}\n                    {{ _('Remove duplicate lines of text') }}\n                
\n                
\n                    {{ render_checkbox_field(form.sort_text_alphabetically) }}\n                    {{ _('Helps reduce changes detected caused by sites shuffling lines around, combine with') }} {{ _('check unique lines') }} {{ _('below.') }}\n                
\n                
\n                    {{ render_checkbox_field(form.trim_text_whitespace) }}\n                    {{ _('Remove any whitespace before and after each line of text') }}\n                
\n                {% include \"edit/text-options.html\" %}\n                
\n              
\n              
\n                \n                
\n                {#
Refresh
#}\n                
\n                  
\n                      
\n                          
{{ _('Loading...') }}
\n                      
\n                      
\n                          
{{ _('Loading...') }}
\n                      
\n                  
\n                
\n                {{ highlight_trigger_ignored_explainer() }}\n            
\n          
\n        
\n\n        {% endif %}\n        {# rendered sub Template #}\n        {% if extra_form_content %}\n            
\n            {{ extra_form_content|safe }}\n            
\n        {% endif %}\n            {% if capabilities.supports_visual_selector %}\n            
\n                \"New\n\n                
\n                    
\n                        {% if capabilities.supports_screenshots and capabilities.supports_xpath_element_data %}\n                            {% if visual_selector_data_ready %}\n                                \n                                    {{ _('The Visual Selector tool lets you select the') }} {{ _('text') }} {{ _('elements that will be used for the change detection. It automatically fills-in the filters in the \"CSS/JSONPath/JQ/XPath Filters\" box of the') }} {{ _('Filters & Triggers') }} {{ _('tab. Use') }} {{ _('Shift+Click') }} {{ _('to select multiple items.') }}\n                                \n\n                                {% if watch['processor'] == 'image_ssim_diff' %} {# @todo, integrate with image_ssim_diff selector better, use some extra form ? #}\n                                
\n                                    \n                                    \n                                    \n                                    {{ render_field(form.processor_config_bounding_box) }}\n                                    {{ render_field(form.processor_config_selection_mode) }}\n                                
\n                                {% endif %}\n\n                                
\n                                    {{ _('Clear selection') }}\n                                    \n                                    {{ _('One moment, fetching screenshot and element information..') }}\n                                
\n                                
\n                                    \n                                    \n                                    \n                                    \n                                    \n                                
\n                                
{{ _('Currently:') }} {{ _('Loading...') }}
\n                            {%  else %}\n                                {{ _('Visual Selector data is not ready, watch needs to be checked atleast once.') }}\n                            {%  endif %}\n                    {% else %}\n                        
\n                            {{ _('Sorry, this functionality only works with fetchers that support Javascript and screenshots (such as playwright etc).') }}
\n                            {{ _('You need to') }} {{ _('Set the fetch method') }} {{ _('to one that supports Javascript and screenshots.') }}\n                        
\n                    {% endif %}\n                    
\n                
\n            
\n            {% endif %}\n            
\n                
\n                    \n                    \n                        \n                        \n                            \n                            \n                        \n                        \n                            \n                            \n                        \n                        \n                            \n                            \n                        \n                        \n                            \n                            \n                        \n                        \n                            \n                            \n                        \n                        \n                            \n                            \n                        \n                        \n                    
{{ _('Check count') }}{{ \"{:,}\".format( watch.check_count) }}
{{ _('Consecutive filter failures') }}{{ \"{:,}\".format( watch.consecutive_filter_failures) }}
{{ _('History length') }}{{ \"{:,}\".format(watch.history|length) }}
{{ _('Last fetch duration') }}{{ watch.fetch_time }}s
{{ _('Notification alert count') }}{{ watch.notification_alert_count }}
{{ _('Server type reply') }}{{ watch.get('remote_server_reply') }}
\n\n                    {% if ui_edit_stats_extras %}\n                    
 \n                        {{ ui_edit_stats_extras|safe }}\n                    
\n                    {% endif %}\n\n                    {% if watch.history_n %}\n                        
\n                             {{ _('Download latest HTML snapshot') }}\n                             {{ _('Download watch data package') }}\n                        
\n                    {% endif %}\n\n                
\n            
\n            \n        
\n    
\n
\n\n{% endblock %}\n"
  },
  {
    "path": "changedetectionio/blueprint/ui/templates/preview.html",
    "content": "{% extends 'base.html' %}\n{% from '_helpers.html' import highlight_trigger_ignored_explainer %}\n{% block content %}\n    \n    \n    \n    \n    \n    {% if versions|length >= 2 %}\n        
\n            
\n                
\n                     \n                    \n                    \n\n                
\n            
\n                
\n                {{ _('Keyboard:') }} \n                ← {{ _('Previous') }}   \n                → {{ _('Next') }}\n        
\n    {% endif %}\n\n    
\n        \n    
\n\n\n    
\n        
\n            
{{ watch.error_text_ctime|format_seconds_ago }} {{ _('seconds ago') }}
\n            
\n            {{ last_error_text }}\n        
\n        
\n\n        
\n            
{{ watch.snapshot_error_screenshot_ctime|format_seconds_ago }} {{ _('seconds ago') }}\n            
\n            \"{{\n        
\n\n        
\n            {{ highlight_trigger_ignored_explainer() }}\n            
{{ current_version|format_timestamp_timeago }}
\n            
{{ content| diff_unescape_difference_spans }}
\n        
\n\n        
\n            
\n                {{ _('For now, Differences are performed on text, not graphically, only the latest screenshot is available.') }}\n            
\n            
\n            {% if capabilities.supports_screenshots %}\n                {% if screenshot %}\n                    
{{ watch.snapshot_screenshot_ctime|format_timestamp_timeago }}
\n                    \"{{\n                {% else %}\n                    {{ _('No screenshot available just yet! Try rechecking the page.') }}\n                {% endif %}\n            {% else %}\n                {{ _('Screenshot requires a Content Fetcher ( Sockpuppetbrowser, selenium, etc ) that supports screenshots.') }}\n            {% endif %}\n        
\n    
\n{% endblock %}\n"
  },
  {
    "path": "changedetectionio/blueprint/ui/views.py",
    "content": "from flask import Blueprint, request, redirect, url_for, flash\nfrom flask_babel import gettext\nfrom changedetectionio.store import ChangeDetectionStore\nfrom changedetectionio.auth_decorator import login_optionally_required\nfrom changedetectionio import worker_pool\n\n\ndef construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMetaData, watch_check_update):\n    views_blueprint = Blueprint('ui_views', __name__, template_folder=\"../ui/templates\")\n\n    @views_blueprint.route(\"/form/add/quickwatch\", methods=['POST'])\n    @login_optionally_required\n    def form_quick_watch_add():\n        from changedetectionio import forms\n        form = forms.quickWatchForm(request.form)\n\n        if not form.validate():\n            for widget, l in form.errors.items():\n                flash(','.join(l), 'error')\n            return redirect(url_for('watchlist.index'))\n\n        url = request.form.get('url').strip()\n        if datastore.url_exists(url):\n            flash(gettext('Warning, URL {} already exists').format(url), \"notice\")\n\n        add_paused = request.form.get('edit_and_watch_submit_button') != None\n        from changedetectionio import processors\n        processor = request.form.get('processor', processors.get_default_processor())\n        new_uuid = datastore.add_watch(url=url, tag=request.form.get('tags','').strip(), extras={'paused': add_paused, 'processor': processor})\n\n        if new_uuid:\n            if add_paused:\n                flash(gettext('Watch added in Paused state, saving will unpause.'))\n                return redirect(url_for('ui.ui_edit.edit_page', uuid=new_uuid, unpause_on_save=1, tag=request.args.get('tag')))\n            else:\n                # Straight into the queue.\n                worker_pool.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': new_uuid}))\n                flash(gettext(\"Watch added.\"))\n\n        return redirect(url_for('watchlist.index', tag=request.args.get('tag','')))\n\n    return views_blueprint\n"
  },
  {
    "path": "changedetectionio/blueprint/watchlist/__init__.py",
    "content": "import os\nimport time\n\nfrom flask import Blueprint, request, make_response, render_template, redirect, url_for, flash, session\nfrom flask_paginate import Pagination, get_page_parameter\nfrom flask_babel import gettext as _\n\nfrom changedetectionio import forms\nfrom changedetectionio import processors\nfrom changedetectionio.store import ChangeDetectionStore\nfrom changedetectionio.auth_decorator import login_optionally_required\n\ndef construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMetaData):\n    watchlist_blueprint = Blueprint('watchlist', __name__, template_folder=\"templates\")\n    \n    @watchlist_blueprint.route(\"/\", methods=['GET'])\n    @login_optionally_required\n    def index():\n        active_tag_req = request.args.get('tag', '').lower().strip()\n        active_tag_uuid = active_tag = None\n\n        # Be sure limit_tag is a uuid\n        if active_tag_req:\n            for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():\n                if active_tag_req == tag.get('title', '').lower().strip() or active_tag_req == uuid:\n                    active_tag = tag\n                    active_tag_uuid = uuid\n                    break\n\n        # Redirect for the old rss path which used the /?rss=true\n        if request.args.get('rss'):\n            return redirect(url_for('rss.feed', tag=active_tag_uuid))\n\n        op = request.args.get('op')\n        if op:\n            uuid = request.args.get('uuid')\n            if op == 'pause':\n                datastore.data['watching'][uuid].toggle_pause()\n            elif op == 'mute':\n                datastore.data['watching'][uuid].toggle_mute()\n\n            datastore.data['watching'][uuid].commit()\n            return redirect(url_for('watchlist.index', tag = active_tag_uuid))\n\n        # Sort by last_changed and add the uuid which is usually the key..\n        sorted_watches = []\n        with_errors = request.args.get('with_errors') == \"1\"\n        unread_only = request.args.get('unread') == \"1\"\n        errored_count = 0\n        search_q = request.args.get('q').strip().lower() if request.args.get('q') else False\n        for uuid, watch in datastore.data['watching'].items():\n            if with_errors and not watch.get('last_error'):\n                continue\n\n            if unread_only and (watch.viewed or watch.last_changed == 0) :\n                continue\n\n            if active_tag_uuid and not active_tag_uuid in watch['tags']:\n                    continue\n            if watch.get('last_error'):\n                errored_count += 1\n\n            if search_q:\n                if (watch.get('title') and search_q in watch.get('title').lower()) or search_q in watch.get('url', '').lower():\n                    sorted_watches.append(watch)\n                elif watch.get('last_error') and search_q in watch.get('last_error').lower():\n                    sorted_watches.append(watch)\n            else:\n                sorted_watches.append(watch)\n\n        form = forms.quickWatchForm(request.form)\n        page = request.args.get(get_page_parameter(), type=int, default=1)\n        total_count = len(sorted_watches)\n\n        pagination = Pagination(page=page,\n                                total=total_count,\n                                per_page=datastore.data['settings']['application'].get('pager_size', 50),\n                                css_framework=\"semantic\",\n                                display_msg=_('displaying {start} - {end} {record_name} in total {total}'),\n                                record_name=_('records'))\n\n        sorted_tags = sorted(datastore.data['settings']['application'].get('tags').items(), key=lambda x: x[1]['title'])\n\n        proxy_list = datastore.proxy_list\n        output = render_template(\n            \"watch-overview.html\",\n            active_tag=active_tag,\n            active_tag_uuid=active_tag_uuid,\n            app_rss_token=datastore.data['settings']['application'].get('rss_access_token'),\n            datastore=datastore,\n            errored_count=errored_count,\n            extra_classes='has-queue' if not update_q.empty() else '',\n            form=form,\n            generate_tag_colors=processors.generate_processor_badge_colors,\n            guid=datastore.data['app_guid'],\n            has_proxies=proxy_list,\n            hosted_sticky=os.getenv(\"SALTED_PASS\", False) == False,\n            now_time_server=round(time.time()),\n            pagination=pagination,\n            processor_badge_css=processors.get_processor_badge_css(),\n            processor_badge_texts=processors.get_processor_badge_texts(),\n            processor_descriptions=processors.get_processor_descriptions(),\n            queue_size=update_q.qsize(),\n            queued_uuids=update_q.get_queued_uuids(),\n            search_q=request.args.get('q', '').strip(),\n            sort_attribute=request.args.get('sort') if request.args.get('sort') else request.cookies.get('sort'),\n            sort_order=request.args.get('order') if request.args.get('order') else request.cookies.get('order'),\n            system_default_fetcher=datastore.data['settings']['application'].get('fetch_backend'),\n            tags=sorted_tags,\n            unread_changes_count=datastore.unread_changes_count,\n            watches=sorted_watches\n        )\n\n        # Return freed template-building memory to the OS immediately.\n        # render_template allocates ~20MB of intermediate strings that are freed on return,\n        # but glibc keeps those pages mapped in its arenas as RSS. malloc_trim() forces\n        # glibc to release them, preventing RSS growth from concurrent Chrome connections.\n        try:\n            import ctypes\n            ctypes.CDLL('libc.so.6').malloc_trim(0)\n        except Exception:\n            pass\n\n        if session.get('share-link'):\n            del (session['share-link'])\n\n        resp = make_response(output)\n\n        # The template can run on cookie or url query info\n        if request.args.get('sort'):\n            resp.set_cookie('sort', request.args.get('sort'))\n        if request.args.get('order'):\n            resp.set_cookie('order', request.args.get('order'))\n\n        return resp\n        \n    return watchlist_blueprint"
  },
  {
    "path": "changedetectionio/blueprint/watchlist/templates/watch-overview.html",
    "content": "{%- extends 'base.html' -%}\n{%- block content -%}\n{%- set tips = [\n    _(\"Changedetection.io can monitor more than just web-pages! See our plugins!\") ~ ' ' ~ _('More info') ~ '',\n    _(\"You can also add 'shared' watches.\") ~ ' ' ~ _('More info') ~ ''\n] -%}\n{%- from '_helpers.html' import render_simple_field, render_field, render_nolabel_field, sort_by_title -%}\n\n\n\n\n\n\n\n
\n\n    
\n        \n        
\n            {{ _('Add a new web page change detection watch') }}\n            
\n                    {{ render_nolabel_field(form.url, placeholder=\"https://...\", required=true) }}\n                    {{ render_nolabel_field(form.watch_submit_button, title=_(\"Watch this URL!\") ) }}\n                    {{ render_nolabel_field(form.edit_and_watch_submit_button, title=_(\"Edit first then Watch\") ) }}\n            
\n            
\n               {{ render_field(form.tags, value=active_tag.title if active_tag_uuid else '', placeholder=_(\"Watch group / tag\"), class=\"transparent-field\") }}\n            
\n            
\n                {{ render_simple_field(form.processor) }}\n            
\n\n        
\n        \n            Tip:  {{ tips | random | safe }}\n        \n    
\n
\n
\n    
\n    \n    \n    
\n        \n        \n        \n        \n        \n        \n        \n        \n        \n        \n        \n    
\n\n    
\n        
{%- if watches|length >= pagination.per_page -%}{{ pagination.info }}{%- endif -%}
\n        
{{ _('Queued size') }}: {{ queue_size }}
\n    
\n\n\n\n    {%- if search_q -%}
{{ _('Searching') }} \"{{search_q}}\"
{%- endif -%}\n    
\n        {{ _('All') }}\n\n    \n    {%- for uuid, tag in tags -%}\n        {%- if tag != \"\" -%}\n            {{ tag.title }}\n        {%- endif -%}\n    {%- endfor -%}\n    
\n\n    {%- set sort_order = sort_order or 'asc' -%}\n    {%- set sort_attribute = sort_attribute or 'last_changed'  -%}\n    {%- set pagination_page = request.args.get('page', 0) -%}\n    {%- set cols_required = 6 -%}\n    {%- set any_has_restock_price_processor = datastore.any_watches_have_processor_by_name(\"restock_diff\") -%}\n    {%- if any_has_restock_price_processor -%}\n        {%- set cols_required = cols_required + 1 -%}\n    {%- endif -%}\n    {%- set ui_settings = datastore.data['settings']['application']['ui'] -%}\n    {%- set wrapper_classes = [\n        'has-unread-changes' if unread_changes_count else '',\n        'has-error' if errored_count else '',\n    ] -%}\n    
\n        {%- set table_classes = [\n            'favicon-enabled' if 'favicons_enabled' not in ui_settings or ui_settings['favicons_enabled'] else 'favicon-not-enabled',\n        ] -%}\n        \n            \n            \n                {%- set link_order = \"desc\" if sort_order  == 'asc' else \"asc\" -%}\n                {%- set arrow_span = \"\" -%}\n                \n                \n                \n             {%- if any_has_restock_price_processor -%}\n                \n             {%- endif -%}\n                \n                \n                \n            \n            \n            \n            {%- if not watches|length -%}\n            \n                \n            \n            {%- endif -%}\n\n            {%- for watch in (watches|sort(attribute=sort_attribute, reverse=sort_order == 'asc'))|pagination_slice(skip=pagination.skip) -%}\n                {%- set checking_now = is_checking_now(watch) -%}\n                {%- set history_n = watch.history_n -%}\n                {%- set favicon = watch.get_favicon_filename() -%}\n                {%- set error_texts = watch.compile_error_texts(has_proxies=has_proxies) -%}\n                {%- set system_use_url_watchlist = datastore.data['settings']['application']['ui'].get('use_page_title_in_list')  -%}\n                {#  Class settings mirrored in changedetectionio/static/js/realtime.js for the frontend #}\n                {%- set row_classes = [\n                    loop.cycle('pure-table-odd', 'pure-table-even'),\n                    'processor-' ~ watch['processor'],\n                    'has-error' if error_texts|length > 2 else '',\n                    'paused' if watch.paused is defined and watch.paused != False else '',\n                    'unviewed' if watch.has_unviewed else '',\n                    'has-restock-info' if watch.has_restock_info else 'no-restock-info',\n                    'has-favicon' if favicon else '',\n                    'in-stock' if watch.has_restock_info and watch['restock']['in_stock'] else '',\n                    'not-in-stock' if watch.has_restock_info and not watch['restock']['in_stock'] else '',\n                    'queued' if watch.uuid in queued_uuids else '',\n                    'checking-now' if checking_now else '',\n                    'notification_muted' if watch.notification_muted else '',\n                    'single-history' if history_n == 1 else '',\n                    'multiple-history' if history_n >= 2 else '',\n                    'use-html-title' if system_use_url_watchlist else 'no-html-title',\n                ] -%}\n            \n                \n                \n\n                \n{%- if any_has_restock_price_processor -%}\n                \n{%- endif -%}\n            {#last_checked becomes fetch-start-time#}\n                \n                \n                \n            \n            {%- endfor -%}\n            \n        
 # \n                    \n                     \n                    \n                {{ _('Website') }} {{ _('Restock & Price') }}{{ _('Last') }} {{ _('Checked') }} {{ _('Last') }} {{ _('Changed') }} 
{{ _('No web page change detection watches configured, please add a URL in the box above, or') }} {{ _('import a list') }}.
 {{ loop.index+pagination.skip }}
\n                    
\n                    \"Pause\n                    \"UnPause\n                    \"Mute\n                    \"UnMute\n                    
\n                		\n                    
\n                    {% if 'favicons_enabled' not in ui_settings or ui_settings['favicons_enabled'] %}\n                        
\n                            {# Intersection Observer lazy loading: store real URL in data-src, load only when visible in viewport #}\n                            \"Favicon\n                        
\n                    {%  endif %}\n                        
\n                            {%- if watch['processor'] and watch['processor'] in processor_badge_texts -%}\n                                {{ processor_badge_texts[watch['processor']] }}\n                            {%- endif -%}\n                            \n                                {% if system_use_url_watchlist or watch.get('use_page_title_in_list') %}\n                                    {{ watch.label }}\n                                {% else %}\n                                    {{ watch.get('title') or watch.link }}\n                                {% endif %}\n                                \n                            \n                            
{{ error_texts|safe }}
\n                            {%- if watch['processor'] == 'text_json_diff'  -%}\n                                {%- if watch['has_ldjson_price_data'] and not watch['track_ldjson_price_data']  -%}\n                                
Switch to Restock & Price watch mode? Yes No
\n                                {%- endif -%}\n                            {%- endif -%}\n\n                            {%- for watch_tag_uuid, watch_tag in datastore.get_all_tags_for_watch(watch['uuid']).items() -%}\n                              {{ watch_tag.title }}\n                            {%- endfor -%}\n                        
\n                    
\n                            \n                            {%- set effective_fetcher = watch.get_fetch_backend if watch.get_fetch_backend != \"system\" else system_default_fetcher -%}\n                            {%- if effective_fetcher and (\"html_webdriver\" in effective_fetcher or \"html_\" in effective_fetcher or \"extra_browser_\" in effective_fetcher) -%}\n                                {{ effective_fetcher|fetcher_status_icons }}\n                            {%- endif -%}\n                            {%- if watch.is_pdf  -%}\"Converting{%- endif -%}\n                            {%- if watch.has_browser_steps -%}\"Browser{%- endif -%}\n\n                    
\n                    
\n                		\n                    {%- if watch['processor'] == 'restock_diff'  -%}\n                        {%- if watch.has_restock_info -%}\n                            \n                                \n                                 {%- if watch['restock']['in_stock']-%}  {{ _('In stock') }} {%- else-%}  {{ _('Not in stock') }} {%- endif -%}\n                            \n                        {%- endif -%}\n\n                        {%- if watch.get('restock') and watch['restock'].get('price') -%}\n                            {%- set restock = watch['restock'] -%}\n                            {%- set price = restock.get('price') -%}\n                            {%- set cur = restock.get('currency','') -%}\n\n                            {%- if price is not none and (price|string)|regex_search('\\d') -%}\n                              \n                              {# @todo: make parse_currency/parse_decimal aware of the locale of the actual web page and use that instead changedetectionio/processors/restock_diff/__init__.py #}\n                                {%- if price is number -%}{# It's a number so we can convert it to their locale' #}\n                                  {{ price|format_number_locale }} {{ cur }}\n                                {%- else -%}{# It's totally fine if it arrives as something else, the website might be something weird in this field #}\n                                  {{ price }} {{ cur }}\n                                {%- endif -%}\n                              \n                            {%- endif -%}\n                        {%- elif not watch.has_restock_info -%}\n                            {{ _('No information') }}\n                        {%- endif -%}\n                    {%- endif -%}\n                \n                    \n                    {{watch|format_last_checked_time|safe}}\n                {%- if watch.history_n >=2 and watch.last_changed >0 -%}\n                    {{watch.last_changed|format_timestamp_timeago}}\n                    {%- else -%}\n                    {{ _('Not yet') }}\n                    {%- endif -%}\n                \n                    
\n                    {%- set target_attr = ' target=\"' ~ watch.uuid ~ '\"' if datastore.data['settings']['application']['ui'].get('open_diff_in_new_tab') else '' -%}\n                    {{ _('Queued') }}\n                    {{ _('Recheck') }}\n                    {{ _('Edit') }}\n                    {{ _('History') }}\n                    {{ _('Preview') }}\n                    
\n                
\n        \n        {{ pagination.links }}\n    
\n    
\n
\n{%- endblock -%}"
  },
  {
    "path": "changedetectionio/browser_steps/__init__.py",
    "content": ""
  },
  {
    "path": "changedetectionio/browser_steps/browser_steps.py",
    "content": "import os\nimport time\nimport re\nfrom random import randint\nfrom loguru import logger\n\nfrom changedetectionio.content_fetchers import SCREENSHOT_MAX_HEIGHT_DEFAULT\nfrom changedetectionio.content_fetchers.base import manage_user_agent\nfrom changedetectionio.jinja2_custom import render as jinja_render\n\ndef browser_steps_get_valid_steps(browser_steps: list):\n    if browser_steps is not None and len(browser_steps):\n        valid_steps = list(filter(\n            lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one'),browser_steps))\n\n        # Just incase they selected Goto site by accident with older JS\n        if valid_steps and valid_steps[0]['operation'] == 'Goto site':\n            del(valid_steps[0])\n\n        return valid_steps\n    return []\n\n\n# Two flags, tell the JS which of the \"Selector\" or \"Value\" field should be enabled in the front end\n# 0- off, 1- on\nbrowser_step_ui_config = {'Choose one': '0 0',\n                          #                 'Check checkbox': '1 0',\n                          #                 'Click button containing text': '0 1',\n                          #                 'Scroll to bottom': '0 0',\n                          #                 'Scroll to element': '1 0',\n                          #                 'Scroll to top': '0 0',\n                          #                 'Switch to iFrame by index number': '0 1'\n                          #                 'Uncheck checkbox': '1 0',\n                          # @todo\n                          'Check checkbox': '1 0',\n                          'Click X,Y': '0 1',\n                          'Click element if exists': '1 0',\n                          'Click element': '1 0',\n                          'Click element containing text': '0 1',\n                          'Click element containing text if exists': '0 1',\n                          'Enter text in field': '1 1',\n                          'Execute JS': '0 1',\n#                          'Extract text and use as filter': '1 0',\n                          'Goto site': '0 0',\n                          'Goto URL': '0 1',\n                          'Make all child elements visible': '1 0',\n                          'Press Enter': '0 0',\n                          'Select by label': '1 1',\n                          ' is rendered\n    def _value(self):\n        # Tag UUID to name, on submit it will convert it back (in the submit handler of init.py)\n        if self.data and type(self.data) is list:\n            tag_titles = []\n            for i in self.data:\n                tag = self.datastore.data['settings']['application']['tags'].get(i)\n                if tag:\n                    tag_title = tag.get('title')\n                    if tag_title:\n                        tag_titles.append(tag_title)\n\n            return ', '.join(tag_titles)\n\n        if not self.data:\n            return ''\n\n        return 'error'\n\nclass TimeDurationForm(Form):\n    hours = SelectField(choices=[(f\"{i}\", f\"{i}\") for i in range(0, 25)], default=\"24\",  validators=[validators.Optional()])\n    minutes = SelectField(choices=[(f\"{i}\", f\"{i}\") for i in range(0, 60)], default=\"00\", validators=[validators.Optional()])\n\nclass TimeStringField(Field):\n    \"\"\"\n    A WTForms field for time inputs (HH:MM) that stores the value as a string.\n    \"\"\"\n    widget = TimeInput()  # Use the built-in time input widget\n\n    def _value(self):\n        \"\"\"\n        Returns the value for rendering in the form.\n        \"\"\"\n        return self.data if self.data is not None else \"\"\n\n    def process_formdata(self, valuelist):\n        \"\"\"\n        Processes the raw input from the form and stores it as a string.\n        \"\"\"\n        if valuelist:\n            time_str = valuelist[0]\n            # Simple validation for HH:MM format\n            if not time_str or len(time_str.split(\":\")) != 2:\n                raise ValidationError(_l(\"Invalid time format. Use HH:MM.\"))\n            self.data = time_str\n\n\nclass validateTimeZoneName(object):\n    \"\"\"\n       Flask wtform validators wont work with basic auth\n    \"\"\"\n\n    def __init__(self, message=None):\n        self.message = message\n\n    def __call__(self, form, field):\n        from zoneinfo import available_timezones\n        python_timezones = available_timezones()\n        if field.data and field.data not in python_timezones:\n            raise ValidationError(_l(\"Not a valid timezone name\"))\n\nclass ScheduleLimitDaySubForm(Form):\n    enabled = BooleanField(_l(\"not set\"), default=True)\n    start_time = TimeStringField(_l(\"Start At\"), default=\"00:00\", validators=[validators.Optional()])\n    duration = FormField(TimeDurationForm, label=_l(\"Run duration\"))\n\nclass ScheduleLimitForm(Form):\n    enabled = BooleanField(_l(\"Use time scheduler\"), default=False)\n    # Because the label for=\"\"\" doesnt line up/work with the actual checkbox\n    monday = FormField(ScheduleLimitDaySubForm, label=\"\")\n    tuesday = FormField(ScheduleLimitDaySubForm, label=\"\")\n    wednesday = FormField(ScheduleLimitDaySubForm, label=\"\")\n    thursday = FormField(ScheduleLimitDaySubForm, label=\"\")\n    friday = FormField(ScheduleLimitDaySubForm, label=\"\")\n    saturday = FormField(ScheduleLimitDaySubForm, label=\"\")\n    sunday = FormField(ScheduleLimitDaySubForm, label=\"\")\n\n    timezone = StringField(_l(\"Optional timezone to run in\"),\n                                  render_kw={\"list\": \"timezones\"},\n                                  validators=[validateTimeZoneName()]\n                                  )\n    def __init__(\n        self,\n        formdata=None,\n        obj=None,\n        prefix=\"\",\n        data=None,\n        meta=None,\n        **kwargs,\n    ):\n        super().__init__(formdata, obj, prefix, data, meta, **kwargs)\n        self.monday.form.enabled.label.text=_l(\"Monday\")\n        self.tuesday.form.enabled.label.text = _l(\"Tuesday\")\n        self.wednesday.form.enabled.label.text = _l(\"Wednesday\")\n        self.thursday.form.enabled.label.text = _l(\"Thursday\")\n        self.friday.form.enabled.label.text = _l(\"Friday\")\n        self.saturday.form.enabled.label.text = _l(\"Saturday\")\n        self.sunday.form.enabled.label.text = _l(\"Sunday\")\n\n\ndef validate_time_between_check_has_values(form):\n    \"\"\"\n    Custom validation function for TimeBetweenCheckForm.\n    Returns True if at least one time interval field has a value > 0.\n    \"\"\"\n    res = any([\n        form.weeks.data and int(form.weeks.data) > 0,\n        form.days.data and int(form.days.data) > 0,\n        form.hours.data and int(form.hours.data) > 0,\n        form.minutes.data and int(form.minutes.data) > 0,\n        form.seconds.data and int(form.seconds.data) > 0\n    ])\n\n    return res\n\n\nclass RequiredTimeInterval(object):\n    \"\"\"\n    WTForms validator that ensures at least one time interval field has a value > 0.\n    Use this with FormField(TimeBetweenCheckForm, validators=[RequiredTimeInterval()]).\n    \"\"\"\n    def __init__(self, message=None):\n        self.message = message or _l('At least one time interval (weeks, days, hours, minutes, or seconds) must be specified.')\n\n    def __call__(self, form, field):\n        if not validate_time_between_check_has_values(field.form):\n            raise ValidationError(self.message)\n\n\nclass TimeBetweenCheckForm(Form):\n    weeks = IntegerField(_l('Weeks'), validators=[validators.Optional(), validators.NumberRange(min=0, message=_l(\"Should contain zero or more seconds\"))])\n    days = IntegerField(_l('Days'), validators=[validators.Optional(), validators.NumberRange(min=0, message=_l(\"Should contain zero or more seconds\"))])\n    hours = IntegerField(_l('Hours'), validators=[validators.Optional(), validators.NumberRange(min=0, message=_l(\"Should contain zero or more seconds\"))])\n    minutes = IntegerField(_l('Minutes'), validators=[validators.Optional(), validators.NumberRange(min=0, message=_l(\"Should contain zero or more seconds\"))])\n    seconds = IntegerField(_l('Seconds'), validators=[validators.Optional(), validators.NumberRange(min=0, message=_l(\"Should contain zero or more seconds\"))])\n    # @todo add total seconds minimum validatior = minimum_seconds_recheck_time\n\n    def __init__(self, formdata=None, obj=None, prefix=\"\", data=None, meta=None, **kwargs):\n        super().__init__(formdata, obj, prefix, data, meta, **kwargs)\n        self.require_at_least_one = kwargs.get('require_at_least_one', False)\n        self.require_at_least_one_message = kwargs.get('require_at_least_one_message', REQUIRE_ATLEAST_ONE_TIME_PART_MESSAGE_DEFAULT)\n\n    def validate(self, **kwargs):\n        \"\"\"Custom validation that can optionally require at least one time interval.\"\"\"\n        # Run normal field validation first\n        if not super().validate(**kwargs):\n            return False\n\n        # Apply optional \"at least one\" validation\n        if self.require_at_least_one:\n            if not validate_time_between_check_has_values(self):\n                # Add error to the form's general errors (not field-specific)\n                if not hasattr(self, '_formdata_errors'):\n                    self._formdata_errors = []\n                self._formdata_errors.append(self.require_at_least_one_message)\n                return False\n\n        return True\n\n\nclass EnhancedFormField(FormField):\n    \"\"\"\n    An enhanced FormField that supports conditional validation with top-level error messages.\n    Adds a 'top_errors' property for validation errors at the FormField level.\n    \"\"\"\n\n    def __init__(self, form_class, label=None, validators=None, separator=\"-\",\n                 conditional_field=None, conditional_message=None, conditional_test_function=None, **kwargs):\n        \"\"\"\n        Initialize EnhancedFormField with optional conditional validation.\n\n        :param conditional_field: Name of the field this FormField depends on (e.g. 'time_between_check_use_default')\n        :param conditional_message: Error message to show when validation fails\n        :param conditional_test_function: Custom function to test if FormField has valid values.\n                                        Should take self.form as parameter and return True if valid.\n        \"\"\"\n        super().__init__(form_class, label, validators, separator, **kwargs)\n        self.top_errors = []\n        self.conditional_field = conditional_field\n        self.conditional_message = conditional_message or \"At least one field must have a value when not using defaults.\"\n        self.conditional_test_function = conditional_test_function\n\n    def validate(self, form, extra_validators=()):\n        \"\"\"\n        Custom validation that supports conditional logic and stores top-level errors.\n        \"\"\"\n        self.top_errors = []\n\n        # First run the normal FormField validation\n        base_valid = super().validate(form, extra_validators)\n\n        # Apply conditional validation if configured\n        if self.conditional_field and hasattr(form, self.conditional_field):\n            conditional_field_obj = getattr(form, self.conditional_field)\n\n            # If the conditional field is False/unchecked, check if this FormField has any values\n            if not conditional_field_obj.data:\n                # Use custom test function if provided, otherwise use generic fallback\n                if self.conditional_test_function:\n                    has_any_value = self.conditional_test_function(self.form)\n                else:\n                    # Generic fallback - check if any field has truthy data\n                    has_any_value = any(field.data for field in self.form if hasattr(field, 'data') and field.data)\n\n                if not has_any_value:\n                    self.top_errors.append(self.conditional_message)\n                    base_valid = False\n\n        return base_valid\n\n\nclass RequiredFormField(FormField):\n    \"\"\"\n    A FormField that passes require_at_least_one=True to TimeBetweenCheckForm.\n    Use this when you want the sub-form to always require at least one value.\n    \"\"\"\n\n    def __init__(self, form_class, label=None, validators=None, separator=\"-\", **kwargs):\n        super().__init__(form_class, label, validators, separator, **kwargs)\n\n    def process(self, formdata, data=unset_value, extra_filters=None):\n        if extra_filters:\n            raise TypeError(\n                \"FormField cannot take filters, as the encapsulated\"\n                \"data is not mutable.\"\n            )\n\n        if data is unset_value:\n            try:\n                data = self.default()\n            except TypeError:\n                data = self.default\n            self._obj = data\n\n        self.object_data = data\n\n        prefix = self.name + self.separator\n        # Pass require_at_least_one=True to the sub-form\n        if isinstance(data, dict):\n            self.form = self.form_class(formdata=formdata, prefix=prefix, require_at_least_one=True, **data)\n        else:\n            self.form = self.form_class(formdata=formdata, obj=data, prefix=prefix, require_at_least_one=True)\n\n    @property\n    def errors(self):\n        \"\"\"Include sub-form validation errors\"\"\"\n        form_errors = self.form.errors\n        # Add any general form errors to a special 'form' key\n        if hasattr(self.form, '_formdata_errors') and self.form._formdata_errors:\n            form_errors = dict(form_errors)  # Make a copy\n            form_errors['form'] = self.form._formdata_errors\n        return form_errors\n\n\n# Separated by  key:value\nclass StringDictKeyValue(StringField):\n    widget = widgets.TextArea()\n\n    def _value(self):\n        if self.data:\n            output = ''\n            for k, v in self.data.items():\n                output += f\"{k}: {v}\\r\\n\"\n            return output\n        else:\n            return ''\n\n    # incoming data processing + validation\n    def process_formdata(self, valuelist):\n        self.data = {}\n        errors = []\n        if valuelist:\n            # Remove empty strings (blank lines)\n            cleaned = [line.strip() for line in valuelist[0].split(\"\\n\") if line.strip()]\n            for idx, s in enumerate(cleaned, start=1):\n                if ':' not in s:\n                    errors.append(f\"Line {idx} is missing a ':' separator.\")\n                    continue\n                parts = s.split(':', 1)\n                key = parts[0].strip()\n                value = parts[1].strip()\n\n                if not key:\n                    errors.append(f\"Line {idx} has an empty key.\")\n                if not value:\n                    errors.append(f\"Line {idx} has an empty value.\")\n\n                self.data[key] = value\n\n        if errors:\n            raise ValidationError(\"Invalid input:\\n\" + \"\\n\".join(errors))\n\nclass ValidateContentFetcherIsReady(object):\n    \"\"\"\n    Validates that anything that looks like a regex passes as a regex\n    \"\"\"\n    def __init__(self, message=None):\n        self.message = message\n\n    def __call__(self, form, field):\n        return\n\n# AttributeError: module 'changedetectionio.content_fetcher' has no attribute 'extra_browser_unlocked<>ASDF213r123r'\n        # Better would be a radiohandler that keeps a reference to each class\n        # if field.data is not None and field.data != 'system':\n        #     klass = getattr(content_fetcher, field.data)\n        #     some_object = klass()\n        #     try:\n        #         ready = some_object.is_ready()\n        #\n        #     except urllib3.exceptions.MaxRetryError as e:\n        #         driver_url = some_object.command_executor\n        #         message = field.gettext('Content fetcher \\'%s\\' did not respond.' % (field.data))\n        #         message += '
' + field.gettext(\n        #             'Be sure that the selenium/webdriver runner is running and accessible via network from this container/host.')\n        #         message += '
' + field.gettext('Did you follow the instructions in the wiki?')\n        #         message += '

' + field.gettext('WebDriver Host: %s' % (driver_url))\n        #         message += '
Go here for more information'\n        #         message += '
'+field.gettext('Content fetcher did not respond properly, unable to use it.\\n %s' % (str(e)))\n        #\n        #         raise ValidationError(message)\n        #\n        #     except Exception as e:\n        #         message = field.gettext('Content fetcher \\'%s\\' did not respond properly, unable to use it.\\n %s')\n        #         raise ValidationError(message % (field.data, e))\n\n\nclass ValidateNotificationBodyAndTitleWhenURLisSet(object):\n    \"\"\"\n       Validates that they entered something in both notification title+body when the URL is set\n       Due to https://github.com/dgtlmoon/changedetection.io/issues/360\n       \"\"\"\n\n    def __init__(self, message=None):\n        self.message = message\n\n    def __call__(self, form, field):\n        if len(field.data):\n            if not len(form.notification_title.data) or not len(form.notification_body.data):\n                message = field.gettext('Notification Body and Title is required when a Notification URL is used')\n                raise ValidationError(message)\n\nclass ValidateAppRiseServers(object):\n    \"\"\"\n       Validates that each URL given is compatible with AppRise\n       \"\"\"\n\n    def __init__(self, message=None):\n        self.message = message\n\n    def __call__(self, form, field):\n        import apprise\n        from .notification.apprise_plugin.assets import apprise_asset\n        from .notification.apprise_plugin.custom_handlers import apprise_http_custom_handler  # noqa: F401\n        from changedetectionio.jinja2_custom import render as jinja_render\n\n        apobj = apprise.Apprise(asset=apprise_asset)\n\n        for server_url in field.data:\n            generic_notification_context_data = NotificationContextData()\n            # Make sure something is atleast in all those regular token fields\n            generic_notification_context_data.set_random_for_validation()\n\n            url = jinja_render(template_str=server_url.strip(), **generic_notification_context_data).strip()\n            if url.startswith(\"#\"):\n                continue\n\n            if not apobj.add(url):\n                message = field.gettext('\\'%s\\' is not a valid AppRise URL.' % (url))\n                raise ValidationError(message)\n\nclass ValidateJinja2Template(object):\n    \"\"\"\n    Validates that a {token} is from a valid set\n    \"\"\"\n    def __call__(self, form, field):\n        from changedetectionio.jinja2_custom import create_jinja_env\n        from jinja2 import BaseLoader, TemplateSyntaxError, UndefinedError\n        from jinja2.meta import find_undeclared_variables\n        import jinja2.exceptions\n\n        # Might be a list of text, or might be just text (like from the apprise url list)\n        joined_data = ' '.join(map(str, field.data)) if isinstance(field.data, list) else f\"{field.data}\"\n\n        try:\n            # Use the shared helper to create a properly configured environment\n            jinja2_env = create_jinja_env(loader=BaseLoader)\n\n            # Add notification tokens for validation\n            static_token_placeholders = NotificationContextData()\n            static_token_placeholders.set_random_for_validation()\n            jinja2_env.globals.update(static_token_placeholders)\n            if hasattr(field, 'extra_notification_tokens'):\n                jinja2_env.globals.update(field.extra_notification_tokens)\n\n            jinja2_env.from_string(joined_data).render()\n        except TemplateSyntaxError as e:\n            raise ValidationError(f\"This is not a valid Jinja2 template: {e}\") from e\n        except UndefinedError as e:\n            raise ValidationError(f\"A variable or function is not defined: {e}\") from e\n        except jinja2.exceptions.SecurityError as e:\n            raise ValidationError(f\"This is not a valid Jinja2 template: {e}\") from e\n\n        # Check for undeclared variables\n        ast = jinja2_env.parse(joined_data)\n        undefined = \", \".join(find_undeclared_variables(ast))\n        if undefined:\n            raise ValidationError(\n                f\"The following tokens used in the notification are not valid: {undefined}\"\n            )\n\nclass validateURL(object):\n\n    \"\"\"\n       Flask wtform validators wont work with basic auth\n    \"\"\"\n\n    def __init__(self, message=None):\n        self.message = message\n\n    def __call__(self, form, field):\n        # This should raise a ValidationError() or not\n        validate_url(field.data)\n\n\ndef validate_url(test_url):\n    from changedetectionio.validate_url import is_safe_valid_url\n    if not is_safe_valid_url(test_url):\n        # This should be wtforms.validators.\n        raise ValidationError('Watch protocol is not permitted or invalid URL format')\n\n\nclass ValidateSinglePythonRegexString(object):\n    def __init__(self, message=None):\n        self.message = message\n\n    def __call__(self, form, field):\n        try:\n            re.compile(field.data)\n        except re.error:\n            message = field.gettext('RegEx \\'%s\\' is not a valid regular expression.')\n            raise ValidationError(message % (field.data))\n\n\nclass ValidateListRegex(object):\n    \"\"\"\n    Validates that anything that looks like a regex passes as a regex\n    \"\"\"\n    def __init__(self, message=None):\n        self.message = message\n\n    def __call__(self, form, field):\n\n        for line in field.data:\n            if re.search(html_tools.PERL_STYLE_REGEX, line, re.IGNORECASE):\n                try:\n                    regex = html_tools.perl_style_slash_enclosed_regex_to_options(line)\n                    re.compile(regex)\n                except re.error:\n                    message = field.gettext('RegEx \\'%s\\' is not a valid regular expression.')\n                    raise ValidationError(message % (line))\n\n\nclass ValidateCSSJSONXPATHInput(object):\n    \"\"\"\n    Filter validation\n    @todo CSS validator ;)\n    \"\"\"\n\n    def __init__(self, message=None, allow_xpath=True, allow_json=True):\n        self.message = message\n        self.allow_xpath = allow_xpath\n        self.allow_json = allow_json\n\n    def __call__(self, form, field):\n\n        if isinstance(field.data, str):\n            data = [field.data]\n        else:\n            data = field.data\n\n        for line in data:\n        # Nothing to see here\n            if not len(line.strip()):\n                return\n\n            # Does it look like XPath?\n            if line.strip()[0] == '/' or line.strip().startswith('xpath:'):\n                if not self.allow_xpath:\n                    raise ValidationError(\"XPath not permitted in this field!\")\n                from lxml import etree, html\n                import elementpath\n                from changedetectionio.html_tools import SafeXPath3Parser\n                tree = html.fromstring(\"\")\n                line = line.replace('xpath:', '')\n\n                try:\n                    elementpath.select(tree, line.strip(), parser=SafeXPath3Parser)\n                except elementpath.ElementPathError as e:\n                    message = field.gettext('\\'%s\\' is not a valid XPath expression. (%s)')\n                    raise ValidationError(message % (line, str(e)))\n                except:\n                    raise ValidationError(\"A system-error occurred when validating your XPath expression\")\n\n            if line.strip().startswith('xpath1:'):\n                if not self.allow_xpath:\n                    raise ValidationError(\"XPath not permitted in this field!\")\n                from lxml import etree, html\n                tree = html.fromstring(\"\")\n                line = re.sub(r'^xpath1:', '', line)\n\n                try:\n                    tree.xpath(line.strip())\n                except etree.XPathEvalError as e:\n                    message = field.gettext('\\'%s\\' is not a valid XPath expression. (%s)')\n                    raise ValidationError(message % (line, str(e)))\n                except:\n                    raise ValidationError(\"A system-error occurred when validating your XPath expression\")\n\n            if 'json:' in line:\n                if not self.allow_json:\n                    raise ValidationError(\"JSONPath not permitted in this field!\")\n\n                from jsonpath_ng.exceptions import (\n                    JsonPathLexerError,\n                    JsonPathParserError,\n                )\n                from jsonpath_ng.ext import parse\n\n                input = line.replace('json:', '')\n\n                try:\n                    parse(input)\n                except (JsonPathParserError, JsonPathLexerError) as e:\n                    message = field.gettext('\\'%s\\' is not a valid JSONPath expression. (%s)')\n                    raise ValidationError(message % (input, str(e)))\n                except:\n                    raise ValidationError(\"A system-error occurred when validating your JSONPath expression\")\n\n                # Re #265 - maybe in the future fetch the page and offer a\n                # warning/notice that its possible the rule doesnt yet match anything?\n                if not self.allow_json:\n                    raise ValidationError(\"jq not permitted in this field!\")\n\n            if 'jq:' in line:\n                try:\n                    import jq\n                except ModuleNotFoundError:\n                    # `jq` requires full compilation in windows and so isn't generally available\n                    raise ValidationError(\"jq not support not found\")\n\n                input = line.replace('jq:', '')\n\n                try:\n                    jq.compile(input)\n                except (ValueError) as e:\n                    message = field.gettext('\\'%s\\' is not a valid jq expression. (%s)')\n                    raise ValidationError(message % (input, str(e)))\n                except:\n                    raise ValidationError(\"A system-error occurred when validating your jq expression\")\n\nclass ValidateSimpleURL:\n    \"\"\"Validate that the value can be parsed by urllib.parse.urlparse() and has a scheme/netloc.\"\"\"\n    def __init__(self, message=None):\n        self.message = message or \"Invalid URL.\"\n\n    def __call__(self, form, field):\n        data = (field.data or \"\").strip()\n        if not data:\n            return  # empty is OK — pair with validators.Optional()\n        from urllib.parse import urlparse\n\n        parsed = urlparse(data)\n        if not parsed.scheme or not parsed.netloc:\n            raise ValidationError(self.message)\n\nclass ValidateStartsWithRegex(object):\n    def __init__(self, regex, *, flags=0, message=None, allow_empty=True, split_lines=True):\n        # compile with given flags (we’ll pass re.IGNORECASE below)\n        self.pattern = re.compile(regex, flags) if isinstance(regex, str) else regex\n        self.message = message\n        self.allow_empty = allow_empty\n        self.split_lines = split_lines\n\n    def __call__(self, form, field):\n        data = field.data\n        if not data:\n            return\n\n        # normalize into list of lines\n        if isinstance(data, str) and self.split_lines:\n            lines = data.splitlines()\n        elif isinstance(data, (list, tuple)):\n            lines = data\n        else:\n            lines = [data]\n\n        for line in lines:\n            stripped = line.strip()\n            if not stripped:\n                if self.allow_empty:\n                    continue\n                raise ValidationError(self.message or _l(\"Empty value not allowed.\"))\n            if not self.pattern.match(stripped):\n                raise ValidationError(self.message or _l(\"Invalid value.\"))\n\nclass quickWatchForm(Form):\n    url = fields.URLField(_l('URL'), validators=[validateURL()])\n    tags = StringTagUUID(_l('Group tag'), validators=[validators.Optional()])\n    watch_submit_button = SubmitField(_l('Watch'), render_kw={\"class\": \"pure-button pure-button-primary\"})\n    processor = RadioField(_l('Processor'), choices=lambda: processors.available_processors(), default=processors.get_default_processor)\n    edit_and_watch_submit_button = SubmitField(_l('Edit > Watch'), render_kw={\"class\": \"pure-button pure-button-primary\"})\n\n\n# Common to a single watch and the global settings\nclass commonSettingsForm(Form):\n    from . import processors\n\n    def __init__(self, formdata=None, obj=None, prefix=\"\", data=None, meta=None, **kwargs):\n        super().__init__(formdata, obj, prefix, data, meta, **kwargs)\n        self.notification_body.extra_notification_tokens = kwargs.get('extra_notification_tokens', {})\n        self.notification_title.extra_notification_tokens = kwargs.get('extra_notification_tokens', {})\n        self.notification_urls.extra_notification_tokens = kwargs.get('extra_notification_tokens', {})\n\n    fetch_backend = RadioField(_l('Fetch Method'), choices=content_fetchers.available_fetchers(), validators=[ValidateContentFetcherIsReady()])\n    notification_body = TextAreaField(_l('Notification Body'), default='{{ watch_url }} had a change.', validators=[validators.Optional(), ValidateJinja2Template()])\n    notification_format = SelectField(_l('Notification format'), choices=list(valid_notification_formats.items()))\n    notification_title = StringField(_l('Notification Title'), default='ChangeDetection.io Notification - {{ watch_url }}', validators=[validators.Optional(), ValidateJinja2Template()])\n    notification_urls = StringListField(_l('Notification URL List'), validators=[validators.Optional(), ValidateAppRiseServers(), ValidateJinja2Template()])\n    processor = RadioField( label=_l(\"Processor - What do you want to achieve?\"), choices=lambda: processors.available_processors(), default=processors.get_default_processor)\n    scheduler_timezone_default = StringField(_l(\"Default timezone for watch check scheduler\"), render_kw={\"list\": \"timezones\"}, validators=[validateTimeZoneName()])\n    webdriver_delay = IntegerField(_l('Wait seconds before extracting text'), validators=[validators.Optional(), validators.NumberRange(min=1, message=_l(\"Should contain one or more seconds\"))])\n\n# Not true anymore but keep the validate_ hook for future use, we convert color tags\n#    def validate_notification_urls(self, field):\n#        \"\"\"Validate that HTML Color format is not used with Telegram\"\"\"\n#        if self.notification_format.data == 'HTML Color' and field.data:\n#            for url in field.data:\n#                if url and ('tgram://' in url or 'discord://' in url or 'discord.com/api/webhooks' in url):\n#                    raise ValidationError('HTML Color format is not supported by Telegram and Discord. Please choose another Notification Format (Plain Text, HTML, or Markdown to HTML).')\n\n\nclass importForm(Form):\n    processor = RadioField(_l('Processor'), choices=lambda: processors.available_processors(), default=processors.get_default_processor)\n    urls = TextAreaField(_l('URLs'))\n    xlsx_file = FileField(_l('Upload .xlsx file'), validators=[FileAllowed(['xlsx'], _l('Must be .xlsx file!'))])\n    file_mapping = SelectField(_l('File mapping'), [validators.DataRequired()], choices={('wachete', 'Wachete mapping'), ('custom','Custom mapping')})\n\nclass SingleBrowserStep(Form):\n\n    operation = SelectField(_l('Operation'), [validators.Optional()], choices=browser_step_ui_config.keys())\n\n    # maybe better to set some  blob.. just return the first that matches json_filter\n    # As a last resort, try to parse the whole \n    soup = BeautifulSoup(content, 'html.parser')\n\n    if ensure_is_ldjson_info_type:\n        bs_result = soup.find_all('script', {\"type\": \"application/ld+json\"})\n    else:\n        bs_result = soup.find_all('script')\n    bs_result += soup.find_all('body')\n\n    bs_jsons = []\n\n    for result in bs_result:\n        # result.text is how bs4 magically strips JSON from the body\n        content_start = result.text.lstrip(\"\\ufeff\").strip()[:100] if result.text else ''\n        # Skip empty tags, and things that dont even look like JSON\n        if not result.text or not (content_start[0] == '{' or content_start[0] == '['):\n            continue\n        try:\n            json_data = json.loads(result.text)\n            bs_jsons.append(json_data)\n        except json.JSONDecodeError:\n            # Skip objects which cannot be parsed\n            continue\n\n    if not bs_jsons:\n        raise JSONNotFound(\"No parsable JSON found in this document\")\n\n    for json_data in bs_jsons:\n        stripped_text_from_html = _parse_json(json_data, json_filter)\n\n        if ensure_is_ldjson_info_type:\n            # Could sometimes be list, string or something else random\n            if isinstance(json_data, dict):\n                # If it has LD JSON 'key' @type, and @type is 'product', and something was found for the search\n                # (Some sites have multiple of the same ld+json @type='product', but some have the review part, some have the 'price' part)\n                # @type could also be a list although non-standard (\"@type\": [\"Product\", \"SubType\"],)\n                # LD_JSON auto-extract also requires some content PLUS the ldjson to be present\n                # 1833 - could be either str or dict, should not be anything else\n\n                t = json_data.get('@type')\n                if t and stripped_text_from_html:\n\n                    if isinstance(t, str) and t.lower() == ensure_is_ldjson_info_type.lower():\n                        break\n                    # The non-standard part, some have a list\n                    elif isinstance(t, list):\n                        if ensure_is_ldjson_info_type.lower() in [x.lower().strip() for x in t]:\n                            break\n\n        elif stripped_text_from_html:\n            break\n\n    return stripped_text_from_html\n\n# content - json\n# json_filter - ie json:$..price\n# ensure_is_ldjson_info_type - str \"product\", optional, \"@type == product\" (I dont know how to do that as a json selector)\ndef extract_json_as_string(content, json_filter, ensure_is_ldjson_info_type=None):\n\n    stripped_text_from_html = False\n# https://github.com/dgtlmoon/changedetection.io/pull/2041#issuecomment-1848397161w\n    # Try to parse/filter out the JSON, if we get some parser error, then maybe it's embedded within HTML tags\n\n    # Looks like clean JSON, dont bother extracting from HTML\n\n    content_start = content.lstrip(\"\\ufeff\").strip()[:100]\n\n    if content_start[0] == '{' or content_start[0] == '[':\n        try:\n            # .lstrip(\"\\ufeff\") strings ByteOrderMark from UTF8 and still lets the UTF work\n            stripped_text_from_html = _parse_json(json.loads(content.lstrip(\"\\ufeff\")), json_filter)\n        except json.JSONDecodeError as e:\n            logger.warning(f\"Error processing JSON {content[:20]}...{str(e)})\")\n    else:\n        # Check for JSONP wrapper: someCallback({...}) or some.namespace({...})\n        # Server may claim application/json but actually return JSONP\n        jsonp_match = re.match(r'^\\w[\\w.]*\\s*\\((.+)\\)\\s*;?\\s*$', content.lstrip(\"\\ufeff\").strip(), re.DOTALL)\n        if jsonp_match:\n            try:\n                inner = jsonp_match.group(1).strip()\n                logger.warning(f\"Content looks like JSONP, attempting to extract inner JSON for filter '{json_filter}'\")\n                stripped_text_from_html = _parse_json(json.loads(inner), json_filter)\n            except json.JSONDecodeError as e:\n                logger.warning(f\"Error processing JSONP inner content {content[:20]}...{str(e)})\")\n\n        if not stripped_text_from_html:\n            # Probably something else, go fish inside for it\n            try:\n                stripped_text_from_html = extract_json_blob_from_html(content=content,\n                                                                      ensure_is_ldjson_info_type=ensure_is_ldjson_info_type,\n                                                                      json_filter=json_filter)\n            except json.JSONDecodeError as e:\n                logger.warning(f\"Error processing JSON while extracting JSON from HTML blob {content[:20]}...{str(e)})\")\n\n    if not stripped_text_from_html:\n        # Re 265 - Just return an empty string when filter not found\n        return ''\n\n    return stripped_text_from_html\n\n# Mode     - \"content\" return the content without the matches (default)\n#          - \"line numbers\" return a list of line numbers that match (int list)\n#\n# wordlist - list of regex's (str) or words (str)\n# Preserves all linefeeds and other whitespacing, its not the job of this to remove that\ndef strip_ignore_text(content, wordlist, mode=\"content\"):\n    ignore_text = []\n    ignore_regex = []\n    ignore_regex_multiline = []\n    ignored_lines = []\n\n    if not content:\n        return ''\n\n    for k in wordlist:\n        # Skip empty strings to avoid matching everything\n        if not k or not k.strip():\n            continue\n        # Is it a regex?\n        res = re.search(PERL_STYLE_REGEX, k, re.IGNORECASE)\n        if res:\n            res = re.compile(perl_style_slash_enclosed_regex_to_options(k))\n            if res.flags & re.DOTALL or res.flags & re.MULTILINE:\n                ignore_regex_multiline.append(res)\n            else:\n                ignore_regex.append(res)\n        else:\n            ignore_text.append(k.strip())\n\n    for r in ignore_regex_multiline:\n        for match in r.finditer(content):\n            content_lines = content[:match.end()].splitlines(keepends=True)\n            match_lines = content[match.start():match.end()].splitlines(keepends=True)\n\n            end_line = len(content_lines)\n            start_line = end_line - len(match_lines)\n\n            if end_line - start_line <= 1:\n                # Match is empty or in the middle of the line\n                ignored_lines.append(start_line)\n            else:\n                for i in range(start_line, end_line):\n                    ignored_lines.append(i)\n\n    line_index = 0\n    lines = content.splitlines(keepends=True)\n    for line in lines:\n        # Always ignore blank lines in this mode. (when this function gets called)\n        got_match = False\n        for l in ignore_text:\n            if l.lower() in line.lower():\n                got_match = True\n\n        if not got_match:\n            for r in ignore_regex:\n                if r.search(line):\n                    got_match = True\n\n        if got_match:\n            ignored_lines.append(line_index)\n\n        line_index += 1\n\n    ignored_lines = set([i for i in ignored_lines if i >= 0 and i < len(lines)])\n\n    # Used for finding out what to highlight\n    if mode == \"line numbers\":\n        return [i + 1 for i in ignored_lines]\n\n    output_lines = set(range(len(lines))) - ignored_lines\n    return ''.join([lines[i] for i in output_lines])\n\ndef cdata_in_document_to_text(html_content: str, render_anchor_tag_content=False) -> str:\n    from xml.sax.saxutils import escape as xml_escape\n    pattern = ')\\s*)*)\\]\\]>'\n    def repl(m):\n        text = m.group(1)\n        return xml_escape(html_to_text(html_content=text)).strip()\n\n    return re.sub(pattern, repl, html_content)\n\n\n# NOTE!! ANYTHING LIBXML, HTML5LIB ETC WILL CAUSE SOME SMALL MEMORY LEAK IN THE LOCAL \"LIB\" IMPLEMENTATION OUTSIDE PYTHON\n\n\ndef html_to_text(html_content: str, render_anchor_tag_content=False, is_rss=False, timeout=10) -> str:\n    \"\"\"\n    Convert HTML content to plain text using inscriptis.\n\n    Thread-Safety: This function uses inscriptis.get_text() which internally calls\n    lxml.html.fromstring() with the default parser. Testing with 50 concurrent threads\n    confirms this approach is thread-safe and produces deterministic output.\n\n    Alternative Approach Rejected: An explicit HTMLParser instance (thread-local or fresh)\n    would also be thread-safe, but was found to break change detection logic in subtle ways\n    (test_check_basic_change_detection_functionality). The default parser provides correct\n    and reliable behavior.\n    \"\"\"\n    from inscriptis import get_text\n    from inscriptis.model.config import ParserConfig\n\n    if render_anchor_tag_content:\n        parser_config = ParserConfig(\n            annotation_rules={\"a\": [\"hyperlink\"]},\n            display_links=True\n        )\n    else:\n        parser_config = None\n    if is_rss:\n        html_content = re.sub(r'])', r'', r'', html_content)\n    else:\n        # Use BS4 html.parser to strip bloat — SPA's often dump 10MB+ of CSS/JS into ,\n        # causing inscriptis to silently give up. Regex-based stripping is unsafe because tags\n        # can appear inside JSON data attributes with JS-escaped closing tags (e.g. <\\/script>),\n        # causing the regex to scan past the intended close and eat real page content.\n        from bs4 import BeautifulSoup\n        soup = BeautifulSoup(html_content, 'html.parser')\n        # Strip tags that inscriptis cannot render as meaningful text and which can be very large.\n        # svg/math: produce path-data/MathML garbage; canvas/iframe/template: no inscriptis handlers.\n        # video/audio/picture are kept — they may contain meaningful fallback text or captions.\n        for tag in soup.find_all(['head', 'script', 'style', 'noscript', 'svg',\n                                  'math', 'canvas', 'iframe', 'template']):\n            tag.decompose()\n\n        # SPAs often use  to hide content until JS loads.\n        # inscriptis respects CSS display rules, so strip hiding styles from the body tag.\n        body_tag = soup.find('body')\n        if body_tag and body_tag.get('style'):\n            style = body_tag['style']\n            if re.search(r'\\b(?:display\\s*:\\s*none|visibility\\s*:\\s*hidden)\\b', style, re.IGNORECASE):\n                logger.debug(f\"html_to_text: Removing hiding styles from body tag (found: '{style}')\")\n                del body_tag['style']\n\n        html_content = str(soup)\n\n    text_content = get_text(html_content, config=parser_config)\n    return text_content\n\n# Does LD+JSON exist with a @type=='product' and a .price set anywhere?\ndef has_ldjson_product_info(content):\n    try:\n        # Better than .lower() which can use a lot of ram\n        if (re.search(r'application/ld\\+json', content, re.IGNORECASE) and\n            re.search(r'\"price\"', content, re.IGNORECASE) and\n            re.search(r'\"pricecurrency\"', content, re.IGNORECASE)):\n            return True\n\n#       On some pages this is really terribly expensive when they dont really need it\n#       (For example you never want price monitoring, but this runs on every watch to suggest it)\n#        for filter in LD_JSON_PRODUCT_OFFER_SELECTORS:\n#            pricing_data += extract_json_as_string(content=content,\n#                                                  json_filter=filter,\n#                                                  ensure_is_ldjson_info_type=\"product\")\n    except Exception as e:\n        # OK too\n        return False\n\n    return False\n\n\n\ndef workarounds_for_obfuscations(content):\n    \"\"\"\n    Some sites are using sneaky tactics to make prices and other information un-renderable by Inscriptis\n    This could go into its own Pip package in the future, for faster updates\n    \"\"\"\n\n    # HomeDepot.com style $90.74\n    # https://github.com/weblyzard/inscriptis/issues/45\n    if not content:\n        return content\n\n    content = re.sub('', '', content)\n\n    return content\n\n\ndef get_triggered_text(content, trigger_text):\n    triggered_text = []\n    result = strip_ignore_text(content=content,\n                               wordlist=trigger_text,\n                               mode=\"line numbers\")\n\n    i = 1\n    for p in content.splitlines():\n        if i in result:\n            triggered_text.append(p)\n        i += 1\n\n    return triggered_text\n\n\ndef extract_title(data: bytes | str, sniff_bytes: int = 2048, scan_chars: int = 8192) -> str | None:\n    try:\n        # Only decode/process the prefix we need for title extraction\n        match data:\n            case bytes() if data.startswith((b\"\\xff\\xfe\", b\"\\xfe\\xff\")):\n                prefix = data[:scan_chars * 2].decode(\"utf-16\", errors=\"replace\")\n            case bytes() if data.startswith((b\"\\xff\\xfe\\x00\\x00\", b\"\\x00\\x00\\xfe\\xff\")):\n                prefix = data[:scan_chars * 4].decode(\"utf-32\", errors=\"replace\")\n            case bytes():\n                try:\n                    prefix = data[:scan_chars].decode(\"utf-8\")\n                except UnicodeDecodeError:\n                    try:\n                        head = data[:sniff_bytes].decode(\"ascii\", errors=\"ignore\")\n                        if m := (META_CS.search(head) or META_CT.search(head)):\n                            enc = m.group(1).lower()\n                        else:\n                            enc = \"cp1252\"\n                        prefix = data[:scan_chars * 2].decode(enc, errors=\"replace\")\n                    except Exception as e:\n                        logger.error(f\"Title extraction encoding detection failed: {e}\")\n                        return None\n            case str():\n                prefix = data[:scan_chars] if len(data) > scan_chars else data\n            case _:\n                logger.error(f\"Title extraction received unsupported data type: {type(data)}\")\n                return None\n\n        # Search only in the prefix\n        if m := TITLE_RE.search(prefix):\n            title = html.unescape(\" \".join(m.group(1).split())).strip()\n            # Some safe limit\n            return title[:2000]\n        return None\n        \n    except Exception as e:\n        logger.error(f\"Title extraction failed: {e}\")\n        return None"
  },
  {
    "path": "changedetectionio/is_safe_url.py",
    "content": "\"\"\"\nURL redirect validation module for preventing open redirect vulnerabilities.\n\nThis module provides functionality to safely validate redirect URLs, ensuring they:\n1. Point to internal routes only (no external redirects)\n2. Are properly normalized (preventing browser parsing differences)\n3. Match registered Flask routes (no fake/non-existent pages)\n4. Are fully logged for security monitoring\n\nReferences:\n- https://flask-login.readthedocs.io/ (safe redirect patterns)\n- https://blog.miguelgrinberg.com/post/the-flask-mega-tutorial-part-v-user-logins\n- https://www.pythonkitchen.com/how-prevent-open-redirect-vulnerab-flask/\n\"\"\"\n\nfrom urllib.parse import urlparse, urljoin\nfrom flask import request\nfrom loguru import logger\n\n\ndef is_safe_url(target, app):\n    \"\"\"\n    Validate that a redirect URL is safe to prevent open redirect vulnerabilities.\n\n    This follows Flask/Werkzeug best practices by ensuring the redirect URL:\n    1. Is a relative path starting with exactly one '/'\n    2. Does not start with '//' (double-slash attack)\n    3. Has no external protocol handlers\n    4. Points to a valid registered route in the application\n    5. Is properly normalized to prevent browser parsing differences\n\n    Args:\n        target: The URL to validate (e.g., '/settings', '/login#top')\n        app: The Flask application instance (needed for route validation)\n\n    Returns:\n        bool: True if the URL is safe for redirection, False otherwise\n\n    Examples:\n        >>> is_safe_url('/settings', app)\n        True\n        >>> is_safe_url('//evil.com', app)\n        False\n        >>> is_safe_url('/settings#general', app)\n        True\n        >>> is_safe_url('/fake-page', app)\n        False\n    \"\"\"\n    if not target:\n        return False\n\n    # Normalize the URL to prevent browser parsing differences\n    # Strip whitespace and replace backslashes (which some browsers interpret as forward slashes)\n    target = target.strip()\n    target = target.replace('\\\\', '/')\n\n    # First, check if it starts with // or more (double-slash attack)\n    if target.startswith('//'):\n        logger.warning(f\"Blocked redirect attempt with double-slash: {target}\")\n        return False\n\n    # Parse the URL to check for scheme and netloc\n    parsed = urlparse(target)\n\n    # Block any URL with a scheme (http://, https://, javascript:, etc.)\n    if parsed.scheme:\n        logger.warning(f\"Blocked redirect attempt with scheme: {target}\")\n        return False\n\n    # Block any URL with a network location (netloc)\n    # This catches patterns like //evil.com, user@host, etc.\n    if parsed.netloc:\n        logger.warning(f\"Blocked redirect attempt with netloc: {target}\")\n        return False\n\n    # At this point, we have a relative URL with no scheme or netloc\n    # Use urljoin to resolve it and verify it points to the same host\n    ref_url = urlparse(request.host_url)\n    test_url = urlparse(urljoin(request.host_url, target))\n\n    # Check: ensure the resolved URL has the same netloc as current host\n    if not (test_url.scheme in ('http', 'https') and ref_url.netloc == test_url.netloc):\n        logger.warning(f\"Blocked redirect attempt with mismatched netloc: {target}\")\n        return False\n\n    # Additional validation: Check if the URL matches a registered route\n    # This prevents redirects to non-existent pages or unintended endpoints\n    try:\n        # Get the path without query string and fragment\n        # Fragments (like #general) are automatically stripped by urlparse\n        path = parsed.path\n\n        # Create a URL adapter bound to the server name\n        adapter = app.url_map.bind(ref_url.netloc)\n\n        # Try to match the path to a registered route\n        # This will raise NotFound if the route doesn't exist\n        endpoint, values = adapter.match(path, return_rule=False)\n\n        # Block redirects to static file endpoints - these are catch-all routes\n        # that would match arbitrary paths, potentially allowing unintended redirects\n        if endpoint in ('static_content', 'static', 'static_flags'):\n            logger.warning(f\"Blocked redirect to static endpoint: {target}\")\n            return False\n\n        # Successfully matched a valid route\n        logger.debug(f\"Validated safe redirect to endpoint '{endpoint}': {target}\")\n        return True\n\n    except Exception as e:\n        # Route doesn't exist or can't be matched\n        logger.warning(f\"Blocked redirect to non-existent route: {target} (error: {e})\")\n        return False\n"
  },
  {
    "path": "changedetectionio/jinja2_custom/__init__.py",
    "content": "\"\"\"\nJinja2 custom extensions and safe rendering utilities.\n\"\"\"\nfrom .extensions.TimeExtension import TimeExtension\nfrom .safe_jinja import (\n    render,\n    render_fully_escaped,\n    create_jinja_env,\n    JINJA2_MAX_RETURN_PAYLOAD_SIZE,\n    DEFAULT_JINJA2_EXTENSIONS,\n)\nfrom .plugins.regex import regex_replace\n\n__all__ = [\n    'TimeExtension',\n    'render',\n    'render_fully_escaped',\n    'create_jinja_env',\n    'JINJA2_MAX_RETURN_PAYLOAD_SIZE',\n    'DEFAULT_JINJA2_EXTENSIONS',\n    'regex_replace',\n]\n"
  },
  {
    "path": "changedetectionio/jinja2_custom/extensions/TimeExtension.py",
    "content": "\"\"\"\nJinja2 TimeExtension - Custom date/time handling for templates.\n\nThis extension provides the {% now %} tag for Jinja2 templates, offering timezone-aware\ndate/time formatting with support for time offsets.\n\nWhy This Extension Exists:\n    The Arrow library has a now() function (arrow.now()), but Jinja2 templates cannot\n    directly call Python functions - they need extensions or filters to expose functionality.\n\n    This TimeExtension serves as a Jinja2-to-Arrow bridge that:\n\n    1. Makes Arrow accessible in templates - Jinja2 requires registering functions/tags\n       through extensions. You cannot use arrow.now() directly in a template.\n\n    2. Provides template-friendly syntax - Instead of complex Python code, you get clean tags:\n       {% now 'UTC' %}\n       {% now 'UTC' + 'hours=2' %}\n       {% now 'Europe/London', '%Y-%m-%d' %}\n\n    3. Adds convenience features on top of Arrow:\n       - Default timezone from environment variable (TZ) or config\n       - Default datetime format configuration\n       - Offset syntax parsing: 'hours=2,minutes=30' → shift(hours=2, minutes=30)\n       - Empty string timezone support to use configured defaults\n\n    4. Maintains security - Works within Jinja2's sandboxed environment so users\n       cannot access arbitrary Python code or objects.\n\n    Essentially, this is a Jinja2 wrapper around arrow.now() and arrow.shift() that\n    provides user-friendly template syntax while maintaining security.\n\nBasic Usage:\n    {% now 'UTC' %}\n    # Output: Wed, 09 Dec 2015 23:33:01\n\nCustom Format:\n    {% now 'UTC', '%Y-%m-%d %H:%M:%S' %}\n    # Output: 2015-12-09 23:33:01\n\nTimezone Support:\n    {% now 'America/New_York' %}\n    {% now 'Europe/London' %}\n    {% now '' %}  # Uses default timezone from environment.default_timezone\n\nTime Offsets (Addition):\n    {% now 'UTC' + 'hours=2' %}\n    {% now 'UTC' + 'hours=2,minutes=30' %}\n    {% now 'UTC' + 'days=1,hours=2,minutes=15,seconds=10' %}\n\nTime Offsets (Subtraction):\n    {% now 'UTC' - 'minutes=11' %}\n    {% now 'UTC' - 'days=2,minutes=33,seconds=1' %}\n\nTime Offsets with Custom Format:\n    {% now 'UTC' + 'hours=2', '%Y-%m-%d %H:%M:%S' %}\n    # Output: 2015-12-10 01:33:01\n\nWeekday Support (for finding next/previous weekday):\n    {% now 'UTC' + 'weekday=0' %}  # Next Monday (0=Monday, 6=Sunday)\n    {% now 'UTC' + 'weekday=4' %}  # Next Friday\n\nConfiguration:\n    - Default timezone: Set via TZ environment variable or override environment.default_timezone\n    - Default format: '%a, %d %b %Y %H:%M:%S' (can be overridden via environment.datetime_format)\n\nEnvironment Customization:\n    from changedetectionio.jinja2_custom import create_jinja_env\n\n    jinja2_env = create_jinja_env()\n    jinja2_env.default_timezone = 'America/New_York'  # Override default timezone\n    jinja2_env.datetime_format = '%Y-%m-%d %H:%M'      # Override default format\n\nSupported Offset Parameters:\n    - years, months, weeks, days\n    - hours, minutes, seconds, microseconds\n    - weekday (0=Monday through 6=Sunday, must be integer)\n\nNote:\n    This extension uses the Arrow library for timezone-aware datetime handling.\n    All timezone names should be valid IANA timezone identifiers (e.g., 'America/New_York').\n\"\"\"\nimport arrow\n\nfrom jinja2 import nodes\nfrom jinja2.ext import Extension\nimport os\n\nclass TimeExtension(Extension):\n    \"\"\"\n    Jinja2 Extension providing the {% now %} tag for timezone-aware date/time rendering.\n\n    This extension adds two attributes to the Jinja2 environment:\n    - datetime_format: Default strftime format string (default: '%a, %d %b %Y %H:%M:%S')\n    - default_timezone: Default timezone for rendering (default: TZ env var or 'UTC')\n\n    Both can be overridden after environment creation by setting the attributes directly.\n    \"\"\"\n\n    tags = {'now'}\n\n    def __init__(self, environment):\n        \"\"\"Jinja2 Extension constructor.\"\"\"\n        super().__init__(environment)\n\n        environment.extend(\n            datetime_format='%a, %d %b %Y %H:%M:%S',\n            default_timezone=os.getenv('TZ', 'UTC').strip()\n        )\n\n    def _datetime(self, timezone, operator, offset, datetime_format):\n        \"\"\"\n        Get current datetime with time offset applied.\n\n        Args:\n            timezone: IANA timezone identifier (e.g., 'UTC', 'America/New_York') or empty string for default\n            operator: '+' for addition or '-' for subtraction\n            offset: Comma-separated offset parameters (e.g., 'hours=2,minutes=30')\n            datetime_format: strftime format string or None to use environment default\n\n        Returns:\n            Formatted datetime string with offset applied\n\n        Example:\n            _datetime('UTC', '+', 'hours=2,minutes=30', '%Y-%m-%d %H:%M:%S')\n            # Returns current time + 2.5 hours\n        \"\"\"\n        # Use default timezone if none specified\n        if not timezone or timezone == '':\n            timezone = self.environment.default_timezone\n\n        d = arrow.now(timezone)\n\n        # parse shift params from offset and include operator\n        shift_params = {}\n        for param in offset.split(','):\n            interval, value = param.split('=')\n            shift_params[interval.strip()] = float(operator + value.strip())\n\n        # Fix weekday parameter can not be float\n        if 'weekday' in shift_params:\n            shift_params['weekday'] = int(shift_params['weekday'])\n\n        d = d.shift(**shift_params)\n\n        if datetime_format is None:\n            datetime_format = self.environment.datetime_format\n        return d.strftime(datetime_format)\n\n    def _now(self, timezone, datetime_format):\n        \"\"\"\n        Get current datetime without any offset.\n\n        Args:\n            timezone: IANA timezone identifier (e.g., 'UTC', 'America/New_York') or empty string for default\n            datetime_format: strftime format string or None to use environment default\n\n        Returns:\n            Formatted datetime string for current time\n\n        Example:\n            _now('America/New_York', '%Y-%m-%d %H:%M:%S')\n            # Returns current time in New York timezone\n        \"\"\"\n        # Use default timezone if none specified\n        if not timezone or timezone == '':\n            timezone = self.environment.default_timezone\n\n        if datetime_format is None:\n            datetime_format = self.environment.datetime_format\n        return arrow.now(timezone).strftime(datetime_format)\n\n    def parse(self, parser):\n        \"\"\"\n        Parse the {% now %} tag and generate appropriate AST nodes.\n\n        This method is called by Jinja2 when it encounters a {% now %} tag.\n        It parses the tag syntax and determines whether to call _now() or _datetime()\n        based on whether offset operations (+ or -) are present.\n\n        Supported syntax:\n            {% now 'timezone' %}                              -> calls _now()\n            {% now 'timezone', 'format' %}                    -> calls _now()\n            {% now 'timezone' + 'offset' %}                   -> calls _datetime()\n            {% now 'timezone' + 'offset', 'format' %}         -> calls _datetime()\n            {% now 'timezone' - 'offset', 'format' %}         -> calls _datetime()\n\n        Args:\n            parser: Jinja2 parser instance\n\n        Returns:\n            nodes.Output: AST output node containing the formatted datetime string\n        \"\"\"\n        lineno = next(parser.stream).lineno\n\n        node = parser.parse_expression()\n\n        if parser.stream.skip_if('comma'):\n            datetime_format = parser.parse_expression()\n        else:\n            datetime_format = nodes.Const(None)\n\n        if isinstance(node, nodes.Add):\n            call_method = self.call_method(\n                '_datetime',\n                [node.left, nodes.Const('+'), node.right, datetime_format],\n                lineno=lineno,\n            )\n        elif isinstance(node, nodes.Sub):\n            call_method = self.call_method(\n                '_datetime',\n                [node.left, nodes.Const('-'), node.right, datetime_format],\n                lineno=lineno,\n            )\n        else:\n            call_method = self.call_method(\n                '_now',\n                [node, datetime_format],\n                lineno=lineno,\n            )\n        return nodes.Output([call_method], lineno=lineno)"
  },
  {
    "path": "changedetectionio/jinja2_custom/extensions/__init__.py",
    "content": ""
  },
  {
    "path": "changedetectionio/jinja2_custom/plugins/__init__.py",
    "content": "\"\"\"\nJinja2 custom filter plugins for changedetection.io\n\"\"\"\nfrom .regex import regex_replace\n\n__all__ = ['regex_replace']\n"
  },
  {
    "path": "changedetectionio/jinja2_custom/plugins/regex.py",
    "content": "\"\"\"\nRegex filter plugin for Jinja2 templates.\n\nProvides regex_replace filter for pattern-based string replacements in templates.\n\"\"\"\nimport re\nimport signal\nfrom loguru import logger\n\n\ndef regex_replace(value: str, pattern: str, replacement: str = '', count: int = 0) -> str:\n    \"\"\"\n    Replace occurrences of a regex pattern in a string.\n\n    Security: Protected against ReDoS (Regular Expression Denial of Service) attacks:\n    - Limits input value size to prevent excessive processing\n    - Uses timeout mechanism to prevent runaway regex operations\n    - Validates pattern complexity to prevent catastrophic backtracking\n\n    Args:\n        value: The input string to perform replacements on\n        pattern: The regex pattern to search for\n        replacement: The replacement string (default: '')\n        count: Maximum number of replacements (0 = replace all, default: 0)\n\n    Returns:\n        String with replacements applied, or original value on error\n\n    Example:\n        {{ \"hello world\" | regex_replace(\"world\", \"universe\") }}\n        {{ diff | regex_replace(\"([^<]+)([^<]+)\", \"Label1: \\\\1\\\\nLabel2: \\\\2\") }}\n\n    Security limits:\n        - Maximum input size: 10MB\n        - Maximum pattern length: 500 characters\n        - Operation timeout: 10 seconds\n        - Dangerous nested quantifier patterns are rejected\n    \"\"\"\n    # Security limits\n    MAX_INPUT_SIZE = 1024 * 1024 * 10 # 10MB max input size\n    MAX_PATTERN_LENGTH = 500  # Maximum regex pattern length\n    REGEX_TIMEOUT_SECONDS = 10  # Maximum time for regex operation\n\n    # Validate input sizes\n    value_str = str(value)\n    if len(value_str) > MAX_INPUT_SIZE:\n        logger.warning(f\"regex_replace: Input too large ({len(value_str)} bytes), truncating\")\n        value_str = value_str[:MAX_INPUT_SIZE]\n\n    if len(pattern) > MAX_PATTERN_LENGTH:\n        logger.warning(f\"regex_replace: Pattern too long ({len(pattern)} chars), rejecting\")\n        return value_str\n\n    # Check for potentially dangerous patterns (basic checks)\n    # Nested quantifiers like (a+)+ can cause catastrophic backtracking\n    dangerous_patterns = [\n        r'\\([^)]*\\+[^)]*\\)\\+',  # (x+)+\n        r'\\([^)]*\\*[^)]*\\)\\+',  # (x*)+\n        r'\\([^)]*\\+[^)]*\\)\\*',  # (x+)*\n        r'\\([^)]*\\*[^)]*\\)\\*',  # (x*)*\n    ]\n\n    for dangerous in dangerous_patterns:\n        if re.search(dangerous, pattern):\n            logger.warning(f\"regex_replace: Potentially dangerous pattern detected: {pattern}\")\n            return value_str\n\n    def timeout_handler(signum, frame):\n        raise TimeoutError(\"Regex operation timed out\")\n\n    try:\n        # Set up timeout for regex operation (Unix-like systems only)\n        # This prevents ReDoS attacks\n        old_handler = None\n        if hasattr(signal, 'SIGALRM'):\n            old_handler = signal.signal(signal.SIGALRM, timeout_handler)\n            signal.alarm(REGEX_TIMEOUT_SECONDS)\n\n        try:\n            result = re.sub(pattern, replacement, value_str, count=count)\n        finally:\n            # Cancel the alarm\n            if hasattr(signal, 'SIGALRM'):\n                signal.alarm(0)\n                if old_handler is not None:\n                    signal.signal(signal.SIGALRM, old_handler)\n\n        return result\n\n    except TimeoutError:\n        logger.error(f\"regex_replace: Regex operation timed out - possible ReDoS attack. Pattern: {pattern}\")\n        return value_str\n    except re.error as e:\n        logger.warning(f\"regex_replace: Invalid regex pattern: {e}\")\n        return value_str\n    except Exception as e:\n        logger.error(f\"regex_replace: Unexpected error: {e}\")\n        return value_str\n"
  },
  {
    "path": "changedetectionio/jinja2_custom/safe_jinja.py",
    "content": "\"\"\"\nSafe Jinja2 render with max payload sizes\n\nSee https://jinja.palletsprojects.com/en/3.1.x/sandbox/#security-considerations\n\"\"\"\n\nimport jinja2.sandbox\nimport typing as t\nimport os\nfrom .extensions.TimeExtension import TimeExtension\nfrom .plugins import regex_replace\n\nJINJA2_MAX_RETURN_PAYLOAD_SIZE = 1024 * int(os.getenv(\"JINJA2_MAX_RETURN_PAYLOAD_SIZE_KB\", 1024 * 10))\n\n# Default extensions - can be overridden in create_jinja_env()\nDEFAULT_JINJA2_EXTENSIONS = [TimeExtension]\n\ndef create_jinja_env(extensions=None, **kwargs) -> jinja2.sandbox.ImmutableSandboxedEnvironment:\n    \"\"\"\n    Create a sandboxed Jinja2 environment with our custom extensions and default timezone.\n\n    Args:\n        extensions: List of extension classes to use (defaults to DEFAULT_JINJA2_EXTENSIONS)\n        **kwargs: Additional arguments to pass to ImmutableSandboxedEnvironment\n\n    Returns:\n        Configured Jinja2 environment\n    \"\"\"\n    if extensions is None:\n        extensions = DEFAULT_JINJA2_EXTENSIONS\n\n    jinja2_env = jinja2.sandbox.ImmutableSandboxedEnvironment(\n        extensions=extensions,\n        **kwargs\n    )\n\n    # Get default timezone from environment variable\n    default_timezone = os.getenv('TZ', 'UTC').strip()\n    jinja2_env.default_timezone = default_timezone\n\n    # Register custom filters\n    jinja2_env.filters['regex_replace'] = regex_replace\n\n    return jinja2_env\n\n\n# This is used for notifications etc, so actually it's OK to send custom HTML such as  etc, but it should limit what data is available.\n# (Which also limits available functions that could be called)\ndef render(template_str, **args: t.Any) -> str:\n    jinja2_env = create_jinja_env()\n    output = jinja2_env.from_string(template_str).render(args)\n    return output[:JINJA2_MAX_RETURN_PAYLOAD_SIZE]\n\ndef render_fully_escaped(content):\n    \"\"\"\n    Escape HTML content safely.\n\n    MEMORY LEAK FIX: Use markupsafe.escape() directly instead of creating\n    Jinja2 environments (was causing 1M+ compilations per page load).\n    Simpler, faster, and no concerns about environment state.\n    \"\"\"\n    from markupsafe import escape\n    return str(escape(content))\n\n"
  },
  {
    "path": "changedetectionio/languages.py",
    "content": "\"\"\"\nLanguage configuration for i18n support\nAutomatically discovers available languages from translations directory\n\"\"\"\nimport os\nfrom pathlib import Path\n\n\ndef get_timeago_locale(flask_locale):\n    \"\"\"\n    Convert Flask-Babel locale codes to timeago library locale codes.\n\n    The Python timeago library (https://github.com/hustcc/timeago) supports 48 locales\n    but uses different naming conventions than Flask-Babel. This function maps between them.\n\n    Notable differences:\n    - Chinese: Flask uses 'zh', timeago uses 'zh_CN'\n    - Portuguese: Flask uses 'pt', timeago uses 'pt_PT' or 'pt_BR'\n    - Swedish: Flask uses 'sv', timeago uses 'sv_SE'\n    - Norwegian: Flask uses 'no', timeago uses 'nb_NO' or 'nn_NO'\n    - Hindi: Flask uses 'hi', timeago uses 'in_HI'\n    - Czech: Flask uses 'cs', but timeago doesn't support Czech - fallback to English\n\n    Args:\n        flask_locale (str): Flask-Babel locale code (e.g., 'cs', 'zh', 'pt')\n\n    Returns:\n        str: timeago library locale code (e.g., 'en', 'zh_CN', 'pt_PT')\n    \"\"\"\n    locale_map = {\n        'zh': 'zh_CN',      # Chinese Simplified\n        # timeago library just hasn't been updated to use the more modern locale naming convention, before BCP 47 / RFC 5646.\n        'zh_TW': 'zh_TW',   # Chinese Traditional (timeago uses zh_TW)\n        'zh_Hant_TW': 'zh_TW',  # Flask-Babel normalizes zh_TW to zh_Hant_TW, map back to timeago's zh_TW\n        'pt': 'pt_PT',      # Portuguese (Portugal)\n        'sv': 'sv_SE',      # Swedish\n        'no': 'nb_NO',      # Norwegian Bokmål\n        'hi': 'in_HI',      # Hindi\n        'cs': 'en',         # Czech not supported by timeago, fallback to English\n        'uk': 'uk',         # Ukrainian\n        'en_GB': 'en',      # British English - timeago uses 'en'\n        'en_US': 'en',      # American English - timeago uses 'en'\n    }\n    return locale_map.get(flask_locale, flask_locale)\n\n# Language metadata: flag icon CSS class and native name\n# Using flag-icons library: https://flagicons.lipis.dev/\nLANGUAGE_DATA = {\n    'en_GB': {'flag': 'fi fi-gb fis', 'name': 'English (UK)'},\n    'en_US': {'flag': 'fi fi-us fis', 'name': 'English (US)'},\n    'de': {'flag': 'fi fi-de fis', 'name': 'Deutsch'},\n    'fr': {'flag': 'fi fi-fr fis', 'name': 'Français'},\n    'ko': {'flag': 'fi fi-kr fis', 'name': '한국어'},\n    'cs': {'flag': 'fi fi-cz fis', 'name': 'Čeština'},\n    'es': {'flag': 'fi fi-es fis', 'name': 'Español'},\n    'pt': {'flag': 'fi fi-pt fis', 'name': 'Português'},\n    'it': {'flag': 'fi fi-it fis', 'name': 'Italiano'},\n    'ja': {'flag': 'fi fi-jp fis', 'name': '日本語'},\n    'zh': {'flag': 'fi fi-cn fis', 'name': '中文 (简体)'},\n    'zh_Hant_TW': {'flag': 'fi fi-tw fis', 'name': '繁體中文'},\n    'ru': {'flag': 'fi fi-ru fis', 'name': 'Русский'},\n    'pl': {'flag': 'fi fi-pl fis', 'name': 'Polski'},\n    'nl': {'flag': 'fi fi-nl fis', 'name': 'Nederlands'},\n    'sv': {'flag': 'fi fi-se fis', 'name': 'Svenska'},\n    'da': {'flag': 'fi fi-dk fis', 'name': 'Dansk'},\n    'no': {'flag': 'fi fi-no fis', 'name': 'Norsk'},\n    'fi': {'flag': 'fi fi-fi fis', 'name': 'Suomi'},\n    'tr': {'flag': 'fi fi-tr fis', 'name': 'Türkçe'},\n    'ar': {'flag': 'fi fi-sa fis', 'name': 'العربية'},\n    'hi': {'flag': 'fi fi-in fis', 'name': 'हिन्दी'},\n    'uk': {'flag': 'fi fi-ua fis', 'name': 'Українська'},\n}\n\n\ndef get_available_languages():\n    \"\"\"\n    Discover available languages by scanning the translations directory\n    Returns a dict of available languages with their metadata\n    \"\"\"\n    translations_dir = Path(__file__).parent / 'translations'\n\n    available = {}\n\n    # Scan for translation directories\n    if translations_dir.exists():\n        for lang_dir in translations_dir.iterdir():\n            if lang_dir.is_dir() and lang_dir.name in LANGUAGE_DATA:\n                # Check if messages.po exists\n                po_file = lang_dir / 'LC_MESSAGES' / 'messages.po'\n                if po_file.exists():\n                    available[lang_dir.name] = LANGUAGE_DATA[lang_dir.name]\n\n    # If no English variants found, fall back to adding en_GB as default\n    if 'en_GB' not in available and 'en_US' not in available:\n        available['en_GB'] = LANGUAGE_DATA['en_GB']\n\n    return available\n\n\ndef get_language_codes():\n    \"\"\"Get list of available language codes\"\"\"\n    return list(get_available_languages().keys())\n\n\ndef get_flag_for_locale(locale):\n    \"\"\"Get flag emoji for a locale, or globe if unknown\"\"\"\n    return LANGUAGE_DATA.get(locale, {}).get('flag', '🌐')\n\n\ndef get_name_for_locale(locale):\n    \"\"\"Get native name for a locale\"\"\"\n    return LANGUAGE_DATA.get(locale, {}).get('name', locale.upper())\n"
  },
  {
    "path": "changedetectionio/model/App.py",
    "content": "from os import getenv\nfrom copy import deepcopy\n\nfrom changedetectionio.blueprint.rss import RSS_FORMAT_TYPES, RSS_CONTENT_FORMAT_DEFAULT\nfrom changedetectionio.model.Tags import TagsDict\n\nfrom changedetectionio.notification import (\n    default_notification_body,\n    default_notification_format,\n    default_notification_title,\n)\n\n# Equal to or greater than this number of FilterNotFoundInResponse exceptions will trigger a filter-not-found notification\n_FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT = 6\nDEFAULT_SETTINGS_HEADERS_USERAGENT='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36'\n\n\n\nclass model(dict):\n    base_config = {\n            'note': \"Hello! If you change this file manually, please be sure to restart your changedetection.io instance!\",\n            'watching': {},\n            'settings': {\n                'headers': {\n                },\n                'requests': {\n                    'extra_proxies': [], # Configurable extra proxies via the UI\n                    'extra_browsers': [],  # Configurable extra proxies via the UI\n                    'jitter_seconds': 0,\n                    'proxy': None, # Preferred proxy connection\n                    'time_between_check': {'weeks': None, 'days': None, 'hours': 3, 'minutes': None, 'seconds': None},\n                    'timeout': int(getenv(\"DEFAULT_SETTINGS_REQUESTS_TIMEOUT\", \"45\")),  # Default 45 seconds\n                    'workers': int(getenv(\"DEFAULT_SETTINGS_REQUESTS_WORKERS\", \"5\")),  # Number of threads, lower is better for slow connections\n                    'default_ua': {\n                        'html_requests': getenv(\"DEFAULT_SETTINGS_HEADERS_USERAGENT\", DEFAULT_SETTINGS_HEADERS_USERAGENT),\n                        'html_webdriver': None,\n                    }\n                },\n                'application': {\n                    # Custom notification content\n                    'all_paused': False,\n                    'all_muted': False,\n                    'api_access_token_enabled': True,\n                    'base_url' : None,\n                    'empty_pages_are_a_change': False,\n                    'fetch_backend': getenv(\"DEFAULT_FETCH_BACKEND\", \"html_requests\"),\n                    'filter_failure_notification_threshold_attempts': _FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT,\n                    'global_ignore_text': [], # List of text to ignore when calculating the comparison checksum\n                    'global_subtractive_selectors': [],\n                    'history_snapshot_max_length': None,\n                    'ignore_whitespace': True,\n                    'ignore_status_codes': False, #@todo implement, as ternary.\n                    'ssim_threshold': '0.96',  # Default SSIM threshold for screenshot comparison\n                    'notification_body': default_notification_body,\n                    'notification_format': default_notification_format,\n                    'notification_title': default_notification_title,\n                    'notification_urls': [], # Apprise URL list\n                    'pager_size': 50,\n                    'password': False,\n                    'render_anchor_tag_content': False,\n                    'rss_access_token': None,\n                    'rss_content_format': RSS_CONTENT_FORMAT_DEFAULT,\n                    'rss_template_type': 'system_default',\n                    'rss_template_override': None,\n                    'rss_diff_length': 5,\n                    'rss_hide_muted_watches': True,\n                    'rss_reader_mode': False,\n                    'scheduler_timezone_default': None,  # Default IANA timezone name\n                    'schema_version' : 0,\n                    'shared_diff_access': False,\n                    'strip_ignored_lines': False,\n                    'tags': None,  # Initialized in __init__ with real datastore_path\n                    'webdriver_delay': None , # Extra delay in seconds before extracting text\n                    'ui': {\n                        'use_page_title_in_list': True,\n                        'open_diff_in_new_tab': True,\n                        'socket_io_enabled': True,\n                        'favicons_enabled': True\n                    },\n                }\n            }\n        }\n\n    def __init__(self, *arg, datastore_path=None, **kw):\n        super(model, self).__init__(*arg, **kw)\n        # Capture any tags data passed in before base_config overwrites the structure\n        existing_tags = self.get('settings', {}).get('application', {}).get('tags') or {}\n        # CRITICAL: deepcopy to avoid sharing mutable objects between instances\n        self.update(deepcopy(self.base_config))\n        # TagsDict requires the real datastore_path at runtime (cannot be set at class-definition time)\n        if datastore_path is None:\n            raise ValueError(\"App.model() requires 'datastore_path' keyword argument\")\n        self['settings']['application']['tags'] = TagsDict(existing_tags, datastore_path=datastore_path)\n\n\ndef parse_headers_from_text_file(filepath):\n    headers = {}\n    with open(filepath, 'r', encoding='utf-8') as f:\n        for l in f.readlines():\n            l = l.strip()\n            if not l.startswith('#') and ':' in l:\n                (k, v) = l.split(':', 1)  # Split only on the first colon\n                headers[k.strip()] = v.strip()\n\n    return headers"
  },
  {
    "path": "changedetectionio/model/Tag.py",
    "content": "\"\"\"\nTag/Group domain model for organizing and overriding watch settings.\n\nARCHITECTURE NOTE: Configuration Override Hierarchy\n===================================================\n\nTags can override Watch settings when overrides_watch=True.\nCurrent implementation requires manual checking in processors:\n\n    for tag_uuid in watch.get('tags'):\n        tag = datastore['settings']['application']['tags'][tag_uuid]\n        if tag.get('overrides_watch'):\n            restock_settings = tag.get('restock_settings', {})\n            break\n\nWith Pydantic, this would be automatic via chain resolution:\n    Watch → Tag (first with overrides_watch) → Global\n\nSee: Watch.py model docstring for full Pydantic architecture explanation\nSee: processors/restock_diff/processor.py:184-192 for current manual implementation\n\"\"\"\n\nfrom changedetectionio.model import watch_base\nfrom changedetectionio.model.persistence import EntityPersistenceMixin\n\nclass model(EntityPersistenceMixin, watch_base):\n    \"\"\"\n    Tag domain model - groups watches and can override their settings.\n\n    Tags inherit from watch_base to reuse all the same fields as Watch.\n    When overrides_watch=True, tag settings take precedence over watch settings\n    for all watches in this tag/group.\n\n    Fields:\n        overrides_watch (bool): If True, this tag's settings override watch settings\n        title (str): Display name for this tag/group\n        uuid (str): Unique identifier\n        ... (all fields from watch_base can be set as tag-level overrides)\n\n    Resolution order when overrides_watch=True:\n        Watch.field → Tag.field (if overrides_watch) → Global.field\n    \"\"\"\n\n    def __init__(self, *arg, **kw):\n        # Parent class (watch_base) handles __datastore and __datastore_path\n        super(model, self).__init__(*arg, **kw)\n\n        self['overrides_watch'] = kw.get('default', {}).get('overrides_watch')\n\n        if kw.get('default'):\n            self.update(kw['default'])\n            del kw['default']\n\n    # _save_to_disk() method provided by EntityPersistenceMixin\n    # commit() and _get_commit_data() methods inherited from watch_base\n    # Tag uses default _get_commit_data() (includes all keys)\n"
  },
  {
    "path": "changedetectionio/model/Tags.py",
    "content": "import os\nimport shutil\nfrom pathlib import Path\nfrom loguru import logger\n\n_SENTINEL = object()\n\n\nclass TagsDict(dict):\n    \"\"\"Dict subclass that removes the corresponding tag.json file when a tag is deleted.\"\"\"\n\n    def __init__(self, *args, datastore_path: str | os.PathLike, **kwargs) -> None:\n        self._datastore_path = Path(datastore_path)\n        super().__init__(*args, **kwargs)\n\n    def __delitem__(self, key: str) -> None:\n        super().__delitem__(key)\n        tag_dir = self._datastore_path / key\n        tag_json_file = tag_dir / \"tag.json\"\n        if not os.path.exists(tag_json_file):\n            logger.critical(f\"Aborting deletion of directory '{tag_dir}' because '{tag_json_file}' does not exist.\")\n            return\n        try:\n            shutil.rmtree(tag_dir)\n            logger.info(f\"Deleted tag directory for tag {key!r}\")\n        except FileNotFoundError:\n            pass\n        except OSError as e:\n            logger.error(f\"Failed to delete tag directory for tag {key!r}: {e}\")\n\n    def pop(self, key: str, default=_SENTINEL):\n        \"\"\"Remove and return tag, deleting its tag.json file. Raises KeyError if missing and no default given.\"\"\"\n        if key in self:\n            value = self[key]\n            del self[key]\n            return value\n        if default is _SENTINEL:\n            raise KeyError(key)\n        return default\n"
  },
  {
    "path": "changedetectionio/model/Watch.py",
    "content": "\"\"\"\nWatch domain model for change detection monitoring.\n\nARCHITECTURE NOTE: Configuration Override Hierarchy\n===================================================\n\nThis module implements Watch objects that inherit from dict (technical debt).\nThe dream architecture would use Pydantic for:\n\n1. CHAIN RESOLUTION (Watch → Tag → Global Settings)\n   - Current: Manual resolution scattered across codebase\n   - Future: @computed_field properties with automatic resolution\n   - Examples: resolved_fetch_backend, resolved_restock_settings, etc.\n\n2. DATABASE BACKEND ABSTRACTION\n   - Current: Domain model tightly coupled to file-based JSON storage\n   - Future: Domain model (Pydantic) separate from persistence layer\n   - Enables: Easy migration to PostgreSQL, MongoDB, etc.\n\n3. TYPE SAFETY & VALIDATION\n   - Current: Dict access with no compile-time checks\n   - Future: Type hints, IDE autocomplete, validation at boundaries\n\nSee class model docstring for detailed explanation and examples.\nSee: processors/restock_diff/processor.py:184-192 for manual resolution example\n\"\"\"\n\nfrom blinker import signal\nfrom changedetectionio.validate_url import is_safe_valid_url\n\nfrom changedetectionio.strtobool import strtobool\nfrom changedetectionio.jinja2_custom import render as jinja_render\nfrom . import watch_base\nfrom .persistence import EntityPersistenceMixin\nimport os\nimport re\nfrom pathlib import Path\nfrom loguru import logger\n\nfrom .. import jinja2_custom as safe_jinja\nfrom ..html_tools import TRANSLATE_WHITESPACE_TABLE\n\nFAVICON_RESAVE_THRESHOLD_SECONDS=86400\nBROTLI_COMPRESS_SIZE_THRESHOLD = int(os.getenv('SNAPSHOT_BROTLI_COMPRESSION_THRESHOLD', 1024*20))\n\n# Module-level favicon filename cache: data_dir → basename (or None)\n# Keyed by data_dir so it survives Watch object recreation, deepcopy, and concurrent requests.\n# Invalidated explicitly in bump_favicon() when a new favicon is saved.\n_FAVICON_FILENAME_CACHE: dict = {}\n\nminimum_seconds_recheck_time = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3))\nmtable = {'seconds': 1, 'minutes': 60, 'hours': 3600, 'days': 86400, 'weeks': 86400 * 7}\n\ndef _brotli_save(contents, filepath, mode=None, fallback_uncompressed=False):\n    \"\"\"\n    Save compressed data using native brotli with streaming compression.\n    Uses chunked compression to minimize peak memory usage and malloc_trim()\n    to force release of C-level memory back to the OS.\n\n    Args:\n        contents: data to compress (str or bytes)\n        filepath: destination file path\n        mode: brotli compression mode (e.g., brotli.MODE_TEXT)\n        fallback_uncompressed: if True, save uncompressed on failure; if False, raise exception\n\n    Returns:\n        str: actual filepath saved (may differ from input if fallback used)\n\n    Raises:\n        Exception: if compression fails and fallback_uncompressed is False\n    \"\"\"\n    import brotli\n    import gc\n    import ctypes\n\n    # Ensure contents are bytes\n    if isinstance(contents, str):\n        contents = contents.encode('utf-8')\n\n    try:\n        original_size = len(contents)\n        logger.debug(f\"Starting brotli streaming compression of {original_size} bytes.\")\n\n        # Create streaming compressor\n        compressor = brotli.Compressor(quality=6, mode=mode if mode is not None else brotli.MODE_GENERIC)\n\n        # Stream compress in chunks to minimize memory usage\n        chunk_size = 65536  # 64KB chunks\n        total_compressed_size = 0\n\n        with open(filepath, 'wb') as f:\n            # Process data in chunks\n            offset = 0\n            while offset < len(contents):\n                chunk = contents[offset:offset + chunk_size]\n                compressed_chunk = compressor.process(chunk)\n                if compressed_chunk:\n                    f.write(compressed_chunk)\n                    total_compressed_size += len(compressed_chunk)\n                offset += chunk_size\n\n            # Finalize compression - critical for proper cleanup\n            final_chunk = compressor.finish()\n            if final_chunk:\n                f.write(final_chunk)\n                total_compressed_size += len(final_chunk)\n\n        logger.debug(f\"Finished brotli compression - From {original_size} to {total_compressed_size} bytes.\")\n\n        # Cleanup: Delete compressor, force Python GC, then force C-level memory release\n        del compressor\n        gc.collect()\n\n        # Force release of C-level memory back to OS (since brotli is a C library)\n        try:\n            ctypes.CDLL('libc.so.6').malloc_trim(0)\n        except Exception:\n            pass  # malloc_trim not available on all systems (e.g., macOS)\n\n        return filepath\n\n    except Exception as e:\n        logger.error(f\"Brotli compression error: {e}\")\n\n        # Compression failed\n        if fallback_uncompressed:\n            logger.warning(f\"Brotli compression failed for {filepath}, saving uncompressed\")\n            fallback_path = filepath.replace('.br', '')\n            with open(fallback_path, 'wb') as f:\n                f.write(contents)\n            return fallback_path\n        else:\n            raise Exception(f\"Brotli compression failed for {filepath}: {e}\")\n\n\nclass model(EntityPersistenceMixin, watch_base):\n    \"\"\"\n    Watch domain model for monitoring URL changes.\n\n    Inherits from watch_base (which inherits dict) - see watch_base docstring for field documentation.\n\n    ## Configuration Override Hierarchy (Chain Resolution)\n\n    The dream architecture uses a 3-level resolution chain:\n        Watch settings → Tag/Group settings → Global settings\n\n    Current implementation is MANUAL (see processor.py:184-192 for example):\n        - Processors manually check watch.get('field')\n        - Then loop through watch.tags to find first tag with overrides_watch=True\n        - Finally fall back to datastore['settings']['application']['field']\n\n    FUTURE: Pydantic-based chain resolution would enable:\n\n        ```python\n        # Instead of manual resolution in every processor:\n        restock_settings = watch.get('restock_settings', {})\n        for tag_uuid in watch.get('tags'):\n            tag = datastore['settings']['application']['tags'][tag_uuid]\n            if tag.get('overrides_watch'):\n                restock_settings = tag.get('restock_settings', {})\n                break\n\n        # Clean computed properties with automatic resolution:\n        @computed_field\n        def resolved_restock_settings(self) -> dict:\n            if self.restock_settings:\n                return self.restock_settings\n            for tag_uuid in self.tags:\n                tag = self._datastore.get_tag(tag_uuid)\n                if tag.overrides_watch and tag.restock_settings:\n                    return tag.restock_settings\n            return self._datastore.settings.restock_settings or {}\n\n        # Usage: watch.resolved_restock_settings (automatic, type-safe, tested once)\n        ```\n\n    Benefits of Pydantic migration:\n        1. Single source of truth for resolution logic (not scattered across processors)\n        2. Type safety + IDE autocomplete (watch.resolved_fetch_backend vs dict navigation)\n        3. Database backend abstraction (domain model separate from persistence)\n        4. Automatic validation at boundaries\n        5. Self-documenting via type hints\n        6. Easy to test resolution independently\n\n    Resolution chain examples that would benefit:\n        - fetch_backend: watch → tag → global (see get_fetch_backend property)\n        - notification_urls: watch → tag → global\n        - time_between_check: watch → global (see threshold_seconds)\n        - restock_settings: watch → tag (see processors/restock_diff/processor.py:184-192)\n        - history_snapshot_max_length: watch → global (see save_history_blob:550-556)\n        - All processor_config_* settings could use tag overrides\n\n    ## Database Backend Abstraction with Pydantic\n\n    Current: Watch inherits dict, tightly coupled to file-based JSON storage\n    Future: Domain model (Watch) separate from persistence layer\n\n        ```python\n        # Domain model (database-agnostic)\n        class Watch(BaseModel):\n            uuid: str\n            url: str\n            # ... validation, business logic\n\n        # Pluggable backends\n        class DataStoreBackend(ABC):\n            def save_watch(self, watch: Watch): ...\n            def load_watch(self, uuid: str) -> Watch: ...\n\n        # Implementations: FileBackend, MongoBackend, PostgresBackend, etc.\n        ```\n\n    This would enable:\n        - Easy migration between storage backends (file → postgres → mongodb)\n        - Pydantic handles serialization/deserialization automatically\n        - Domain logic stays clean (no storage concerns in Watch methods)\n\n    ## Migration Path\n\n    Given existing codebase, incremental migration recommended:\n        1. Create Pydantic models alongside existing dict-based models\n        2. Add .to_pydantic() / .from_pydantic() bridge methods\n        3. Gradually migrate code to use Pydantic models\n        4. Remove dict inheritance once migration complete\n\n    See: watch_base docstring for technical debt discussion\n    See: processors/restock_diff/processor.py:184-192 for manual resolution example\n    See: Watch.py:550-556 for nested dict navigation that would become watch.resolved_*\n    \"\"\"\n    __newest_history_key = None\n    __history_n = 0\n    jitter_seconds = 0\n\n    def __init__(self, *arg, **kw):\n        # Validate __datastore before calling parent (Watch requires it)\n        if not kw.get('__datastore'):\n            raise ValueError(\"Watch object requires '__datastore' reference - cannot access global settings without it\")\n\n        # Parent class (watch_base) handles __datastore and __datastore_path\n        super(model, self).__init__(*arg, **kw)\n\n        if kw.get('default'):\n            self.update(kw['default'])\n            del kw['default']\n\n        if self.get('default'):\n            del self['default']\n\n        # Be sure the cached timestamp is ready\n        bump = self.history\n\n    # Note: __deepcopy__, __getstate__, and __setstate__ are inherited from watch_base\n    # This prevents memory leaks by sharing __datastore reference instead of copying it\n\n    @property\n    def viewed(self):\n        # Don't return viewed when last_viewed is 0 and newest_key is 0\n        if int(self['last_viewed']) and int(self['last_viewed']) >= int(self.newest_history_key) :\n            return True\n\n        return False\n\n    @property\n    def has_unviewed(self):\n        return int(self.newest_history_key) > int(self['last_viewed']) and self.__history_n >= 2\n\n    @property\n    def link(self):\n\n        url = self.get('url', '')\n        if not is_safe_valid_url(url):\n            return 'DISABLED'\n\n        ready_url = url\n        if '{%' in url or '{{' in url:\n            # Jinja2 available in URLs along with https://pypi.org/project/jinja2-time/\n            try:\n                ready_url = jinja_render(template_str=url)\n            except Exception as e:\n                logger.critical(f\"Invalid URL template for: '{url}' - {str(e)}\")\n                from flask import flash, url_for\n                from markupsafe import Markup\n                message = Markup('The URL {} is invalid and cannot be used, click to edit'.format(\n                    url_for('ui.ui_edit.edit_page', uuid=self.get('uuid')), self.get('url', '')))\n                flash(message, 'error')\n                return ''\n\n        if ready_url.startswith('source:'):\n            ready_url=ready_url.replace('source:', '')\n\n        # Also double check it after any Jinja2 formatting just incase\n        if not is_safe_valid_url(ready_url):\n            return 'DISABLED'\n        return ready_url\n\n    @property\n    def domain_only_from_link(self):\n        from urllib.parse import urlparse\n        parsed = urlparse(self.link)\n        domain = parsed.hostname\n        return domain\n\n    @property\n    def history_index_filename(self):\n        # So that you dont try to view different histories in different 'diff' setups, can confuse cdio.\n        processor = self.get('processor')\n        if not processor or self.get('processor') == 'text_json_diff':\n            return 'history.txt'\n        else:\n            return f'history-{processor}.txt'\n\n    def clear_watch(self):\n        import pathlib\n\n        # Get list of processor config files to preserve\n        from changedetectionio.processors import find_processors\n        processor_names = [name for cls, name in find_processors()]\n        processor_config_files = {f\"{name}.json\" for name in processor_names}\n\n        # JSON Data, Screenshots, Textfiles (history index and snapshots), HTML in the future etc\n        # But preserve processor config files (they're configuration, not history data)\n        # Use glob not rglob here for safety.\n        for item in pathlib.Path(str(self.data_dir)).glob(\"*.*\"):\n            # Skip processor config files\n            if item.name in processor_config_files:\n                continue\n            os.unlink(item)\n\n        # Force the attr to recalculate\n        bump = self.history\n\n        # Do this last because it will trigger a recheck due to last_checked being zero\n        self.update({\n            'browser_steps_last_error_step': None,\n            'check_count': 0,\n            'fetch_time': 0.0,\n            'has_ldjson_price_data': None,\n            'last_checked': 0,\n            'last_error': False,\n            'last_notification_error': False,\n            'last_viewed': 0,\n            'previous_md5': False,\n            'remote_server_reply': None,\n            'track_ldjson_price_data': None\n        })\n        watch_check_update = signal('watch_check_update')\n        if watch_check_update:\n            watch_check_update.send(watch_uuid=self.get('uuid'))\n\n        return\n\n    @property\n    def is_source_type_url(self):\n        return self.get('url', '').startswith('source:')\n\n    @property\n    def get_fetch_backend(self):\n        \"\"\"\n        Get the fetch backend for this watch with special case handling.\n\n        CHAIN RESOLUTION OPPORTUNITY:\n        Currently returns watch.fetch_backend directly, but doesn't implement\n        Watch → Tag → Global resolution chain. With Pydantic:\n\n        @computed_field\n        def resolved_fetch_backend(self) -> str:\n            # Special case: PDFs always use html_requests\n            if self.is_pdf:\n                return 'html_requests'\n\n            # Watch override\n            if self.fetch_backend and self.fetch_backend != 'system':\n                return self.fetch_backend\n\n            # Tag override (first tag with overrides_watch=True wins)\n            for tag_uuid in self.tags:\n                tag = self._datastore.get_tag(tag_uuid)\n                if tag.overrides_watch and tag.fetch_backend:\n                    return tag.fetch_backend\n\n            # Global default\n            return self._datastore.settings.fetch_backend\n        \"\"\"\n        # Maybe also if is_image etc?\n        # This is because chrome/playwright wont render the PDF in the browser and we will just fetch it and use pdf2html to see the text.\n        if self.is_pdf:\n            return 'html_requests'\n\n        return self.get('fetch_backend')\n\n    @property\n    def fetcher_supports_screenshots(self):\n        \"\"\"Return True if the fetcher configured for this watch supports screenshots.\n\n        Resolves 'system' via self._datastore, then checks supports_screenshots on\n        the actual fetcher class. Works for built-in and plugin fetchers alike.\n        \"\"\"\n        from changedetectionio import content_fetchers\n\n        fetcher_name = self.get_fetch_backend  # already handles is_pdf → html_requests\n        if not fetcher_name or fetcher_name == 'system':\n            fetcher_name = self._datastore['settings']['application'].get('fetch_backend', 'html_requests')\n\n        fetcher_class = getattr(content_fetchers, fetcher_name, None)\n        if fetcher_class is None:\n            return False\n\n        return bool(getattr(fetcher_class, 'supports_screenshots', False))\n\n    @property\n    def is_pdf(self):\n        url = str(self.get(\"url\") or \"\").lower()\n        content_type = str(self.get(\"content-type\") or \"\").lower()\n\n        if content_type in (\"none\", \"null\", \"\"):\n            content_type = \"\"\n\n        return (\n                url.endswith(\".pdf\")\n                or content_type.split(\";\")[0].strip() == \"application/pdf\"\n        )\n\n    @property\n    def label(self):\n        # Used for sorting, display, etc\n        return self.get('title') or self.get('page_title') or self.link\n\n    @property\n    def last_changed(self):\n        # last_changed will be the newest snapshot, but when we have just one snapshot, it should be 0\n        if self.__history_n <= 1:\n            return 0\n        if self.__newest_history_key:\n            return int(self.__newest_history_key)\n        return 0\n\n    @property\n    def history_n(self):\n        return self.__history_n\n\n    @property\n    def history(self):\n        \"\"\"History index is just a text file as a list\n            {watch-uuid}/history.txt\n\n            contains a list like\n\n            {epoch-time},{filename}\\n\n\n            We read in this list as the history information\n\n        \"\"\"\n        tmp_history = {}\n\n        # In the case we are only using the watch for processing without history\n        if not self.data_dir:\n            return []\n\n        # Read the history file as a dict\n        fname = os.path.join(self.data_dir, self.history_index_filename)\n        if os.path.isfile(fname):\n            logger.debug(f\"Reading watch history index for {self.get('uuid')}\")\n            with open(fname, \"r\", encoding='utf-8') as f:\n                for i in f.readlines():\n                    if ',' in i:\n                        k, v = i.strip().split(',', 2)\n\n                        # The index history could contain a relative path, so we need to make the fullpath\n                        # so that python can read it\n                        # Cross-platform: check for any path separator (works on Windows and Unix)\n                        if os.sep not in v and '/' not in v and '\\\\' not in v:\n                            # Relative filename only, no path separators\n                            v = os.path.join(self.data_dir, v)\n                        else:\n                            # It's possible that they moved the datadir on older versions\n                            # So the snapshot exists but is in a different path\n                            # Cross-platform: use os.path.basename instead of split('/')\n                            snapshot_fname = os.path.basename(v)\n                            proposed_new_path = os.path.join(self.data_dir, snapshot_fname)\n                            if not os.path.exists(v) and os.path.exists(proposed_new_path):\n                                v = proposed_new_path\n\n                        tmp_history[k] = v\n\n        if len(tmp_history):\n            self.__newest_history_key = list(tmp_history.keys())[-1]\n        else:\n            self.__newest_history_key = None\n\n        self.__history_n = len(tmp_history)\n\n        return tmp_history\n\n    @property\n    def has_history(self):\n        fname = os.path.join(self.data_dir, self.history_index_filename)\n        return os.path.isfile(fname)\n\n    @property\n    def has_browser_steps(self):\n        has_browser_steps = self.get('browser_steps') and list(filter(\n            lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one' and s['operation'] != 'Goto site'),\n            self.get('browser_steps')))\n\n        return has_browser_steps\n\n    @property\n    def has_restock_info(self):\n        if self.get('restock') and self['restock'].get('in_stock') != None:\n                return True\n\n        return False\n\n    # Returns the newest key, but if theres only 1 record, then it's counted as not being new, so return 0.\n    @property\n    def newest_history_key(self):\n        if self.__newest_history_key is not None:\n            return self.__newest_history_key\n\n        if len(self.history) <= 1:\n            return 0\n\n\n        bump = self.history\n        return self.__newest_history_key\n\n    # Given an arbitrary timestamp, find the best history key for the [diff] button so it can preset a smarter from_version\n    @property\n    def get_from_version_based_on_last_viewed(self):\n\n        \"\"\"Unfortunately for now timestamp is stored as string key\"\"\"\n        keys = list(self.history.keys())\n        if not keys:\n            return None\n        if len(keys) == 1:\n            return keys[0]\n\n        last_viewed = int(self.get('last_viewed'))\n        sorted_keys = sorted(keys, key=lambda x: int(x))\n        sorted_keys.reverse()\n\n        # When the 'last viewed' timestamp is greater than or equal the newest snapshot, return second newest\n        if last_viewed >= int(sorted_keys[0]):\n            return sorted_keys[1]\n        \n        # When the 'last viewed' timestamp is between snapshots, return the older snapshot\n        for newer, older in list(zip(sorted_keys[0:], sorted_keys[1:])):\n            if last_viewed < int(newer) and last_viewed >= int(older):\n                return older\n\n        # When the 'last viewed' timestamp is less than the oldest snapshot, return oldest\n        return sorted_keys[-1]\n\n    def get_history_snapshot(self, timestamp=None, filepath=None):\n        \"\"\"\n        Accepts either timestamp or filepath\n        :param timestamp:\n        :param filepath:\n        :return:\n        \"\"\"\n        import brotli\n\n        if not filepath:\n            filepath = self.history[timestamp]\n\n        # Check if binary file (image, PDF, etc.)\n        # Binary files are NEVER saved with .br compression, only text files are\n        binary_extensions = ('.png', '.jpg', '.jpeg', '.gif', '.webp', '.pdf', '.bin', '.jfif')\n        is_binary = any(filepath.endswith(ext) for ext in binary_extensions)\n\n        # Only look for .br versions for text files\n        if not is_binary:\n            # See if a brotli version exists and switch to that (text files only)\n            if not filepath.endswith('.br') and os.path.isfile(f\"{filepath}.br\"):\n                filepath = f\"{filepath}.br\"\n\n            # OR in the backup case that the .br does not exist, but the plain one does\n            if filepath.endswith('.br') and not os.path.isfile(filepath):\n                if os.path.isfile(filepath.replace('.br', '')):\n                    filepath = filepath.replace('.br', '')\n\n        # Handle .br compressed text files\n        if filepath.endswith('.br'):\n            # Brotli doesnt have a fileheader to detect it, so we rely on filename\n            # https://www.rfc-editor.org/rfc/rfc7932\n            # Note: .br should ONLY exist for text files, never binary\n            with open(filepath, 'rb') as f:\n                return brotli.decompress(f.read()).decode('utf-8')\n\n        # Binary file - return raw bytes\n        if is_binary:\n            with open(filepath, 'rb') as f:\n                return f.read()\n\n        # Text file - decode to string\n        with open(filepath, 'r', encoding='utf-8', errors='ignore') as f:\n            return f.read()\n\n    def _write_atomic(self, dest, data, mode='wb'):\n        \"\"\"Write data atomically to dest using a temp file\"\"\"\n        import tempfile\n        with tempfile.NamedTemporaryFile(mode, delete=False, dir=self.data_dir) as tmp:\n            tmp.write(data)\n            tmp.flush()\n            os.fsync(tmp.fileno())\n            tmp_path = tmp.name\n        os.replace(tmp_path, dest)\n\n    def history_trim(self, newest_n_items):\n        from pathlib import Path\n        import gc\n        # Sort by timestamp (key)\n        sorted_items = sorted(self.history.items(), key=lambda x: int(x[0]))\n\n        keep_part = dict(sorted_items[-newest_n_items:])\n        delete_part = dict(sorted_items[:-newest_n_items])\n        logger.info( f\"[{self.get('uuid')}] Trimming history to most recent {newest_n_items} items, keeping {len(keep_part)} items deleting {len(delete_part)} items.\")\n\n        if delete_part:\n            for item in delete_part.items():\n                try:\n                    Path(item[1]).unlink(missing_ok=True)\n                except Exception as e:\n                    logger.critical(f\"{str(e)}\")\n                finally:\n                    logger.debug(f\"[{self.get('uuid')}] Deleted {item[1]} history snapshot\")\n        try:\n            dest = os.path.join(self.data_dir, self.history_index_filename)\n            output = \"\\r\\n\".join(\n                f\"{k},{Path(v).name}\"\n                for k, v in keep_part.items()\n            )+\"\\r\\n\"\n            self._write_atomic(dest=dest, data=output, mode='w')\n        except Exception as e:\n            logger.critical(f\"{str(e)}\")\n        finally:\n            logger.debug(f\"[{self.get('uuid')}] Updated history index {dest}\")\n\n        # reimport\n        bump = self.history\n        gc.collect()\n\n    # Save some text file to the appropriate path and bump the history\n    # result_obj from fetch_site_status.run()\n    def save_history_blob(self, contents, timestamp, snapshot_id):\n\n        logger.trace(f\"{self.get('uuid')} - Updating {self.history_index_filename} with timestamp {timestamp}\")\n\n        self.ensure_data_dir_exists()\n        skip_brotli = strtobool(os.getenv('DISABLE_BROTLI_TEXT_SNAPSHOT', 'False'))\n\n        # Binary data - detect file type and save without compression\n        if isinstance(contents, bytes):\n            try:\n                import puremagic\n                detections = puremagic.magic_string(contents[:2048])\n                ext = detections[0].extension if detections else 'bin'\n                # Strip leading dot if present (puremagic returns extensions like '.jfif')\n                ext = ext.lstrip('.')\n                if detections:\n                    logger.trace(f\"Detected file type: {detections[0].mime_type} -> extension: {ext}\")\n            except Exception as e:\n                logger.warning(f\"puremagic detection failed: {e}, using 'bin' extension\")\n                ext = 'bin'\n\n            snapshot_fname = f\"{snapshot_id}.{ext}\"\n            dest = os.path.join(self.data_dir, snapshot_fname)\n            self._write_atomic(dest, contents)\n            logger.trace(f\"Saved binary snapshot as {snapshot_fname} ({len(contents)} bytes)\")\n\n        # Text data - use brotli compression if enabled and above threshold\n        else:\n            if not skip_brotli and len(contents) > BROTLI_COMPRESS_SIZE_THRESHOLD:\n                # Compressed text\n                import brotli\n                snapshot_fname = f\"{snapshot_id}.txt.br\"\n                dest = os.path.join(self.data_dir, snapshot_fname)\n\n                if not os.path.exists(dest):\n                    try:\n                        actual_dest = _brotli_save(contents, dest, mode=brotli.MODE_TEXT, fallback_uncompressed=True)\n                        if actual_dest != dest:\n                            snapshot_fname = os.path.basename(actual_dest)\n                    except Exception as e:\n                        logger.error(f\"{self.get('uuid')} - Brotli compression failed: {e}\")\n                        # Fallback to uncompressed\n                        snapshot_fname = f\"{snapshot_id}.txt\"\n                        dest = os.path.join(self.data_dir, snapshot_fname)\n                        self._write_atomic(dest, contents.encode('utf-8'))\n            else:\n                # Plain text\n                snapshot_fname = f\"{snapshot_id}.txt\"\n                dest = os.path.join(self.data_dir, snapshot_fname)\n                self._write_atomic(dest, contents.encode('utf-8'))\n\n        # Append to history.txt atomically\n        index_fname = os.path.join(self.data_dir, self.history_index_filename)\n        index_line = f\"{timestamp},{snapshot_fname}\\n\"\n\n        with open(index_fname, 'a', encoding='utf-8') as f:\n            f.write(index_line)\n            f.flush()\n            os.fsync(f.fileno())\n\n        # Update internal state\n        self.__newest_history_key = timestamp\n        self.__history_n += 1\n\n        # MANUAL CHAIN RESOLUTION: Watch → Global\n        # With Pydantic, this would become: maxlen = watch.resolved_history_snapshot_max_length\n        # @computed_field def resolved_history_snapshot_max_length(self) -> Optional[int]:\n        #     if self.history_snapshot_max_length: return self.history_snapshot_max_length\n        #     if tag := self._get_override_tag(): return tag.history_snapshot_max_length\n        #     return self._datastore.settings.history_snapshot_max_length\n        maxlen = self.get('history_snapshot_max_length') or self.get_global_setting('application', 'history_snapshot_max_length')\n\n        if maxlen and self.__history_n and self.__history_n > maxlen:\n            self.history_trim(newest_n_items=maxlen)\n\n        # @todo bump static cache of the last timestamp so we dont need to examine the file to set a proper ''viewed'' status\n        return snapshot_fname\n\n    @property\n    def has_empty_checktime(self):\n        # using all() + dictionary comprehension\n        # Check if all values are 0 in dictionary\n        res = all(x == None or x == False or x==0 for x in self.get('time_between_check', {}).values())\n        return res\n\n    def threshold_seconds(self):\n        seconds = 0\n        for m, n in mtable.items():\n            x = self.get('time_between_check', {}).get(m, None)\n            if x:\n                seconds += x * n\n        return seconds\n\n    # Iterate over all history texts and see if something new exists\n    # Always applying .strip() to start/end but optionally replace any other whitespace\n    def lines_contain_something_unique_compared_to_history(self, lines: list, ignore_whitespace=False):\n        local_lines = set([])\n        if lines:\n            if ignore_whitespace:\n                if isinstance(lines[0], str): # Can be either str or bytes depending on what was on the disk\n                    local_lines = set([l.translate(TRANSLATE_WHITESPACE_TABLE).lower() for l in lines])\n                else:\n                    local_lines = set([l.decode('utf-8').translate(TRANSLATE_WHITESPACE_TABLE).lower() for l in lines])\n            else:\n                if isinstance(lines[0], str): # Can be either str or bytes depending on what was on the disk\n                    local_lines = set([l.strip().lower() for l in lines])\n                else:\n                    local_lines = set([l.decode('utf-8').strip().lower() for l in lines])\n\n\n        # Compare each lines (set) against each history text file (set) looking for something new..\n        existing_history = set({})\n        for k, v in self.history.items():\n            content = self.get_history_snapshot(filepath=v)\n\n            if ignore_whitespace:\n                alist = set([line.translate(TRANSLATE_WHITESPACE_TABLE).lower() for line in content.splitlines()])\n            else:\n                alist = set([line.strip().lower() for line in content.splitlines()])\n\n            existing_history = existing_history.union(alist)\n\n        # Check that everything in local_lines(new stuff) already exists in existing_history - it should\n        # if not, something new happened\n        return not local_lines.issubset(existing_history)\n\n    def get_screenshot(self):\n        fname = os.path.join(self.data_dir, \"last-screenshot.png\")\n        if os.path.isfile(fname):\n            return fname\n\n        # False is not an option for AppRise, must be type None\n        return None\n\n    def favicon_is_expired(self):\n        favicon_fname = self.get_favicon_filename()\n        import glob\n        import time\n\n        if not favicon_fname:\n            return True\n        try:\n            fname = next(iter(glob.glob(os.path.join(self.data_dir, \"favicon.*\"))), None)\n            logger.trace(f\"Favicon file maybe found at {fname}\")\n            if os.path.isfile(fname):\n                file_age = int(time.time() - os.path.getmtime(fname))\n                logger.trace(f\"Favicon file age is {file_age}s\")\n                if file_age < FAVICON_RESAVE_THRESHOLD_SECONDS:\n                    return False\n        except Exception as e:\n            logger.critical(f\"Exception checking Favicon age {str(e)}\")\n            return True\n\n        # Also in the case that the file didnt exist\n        return True\n\n    def bump_favicon(self, url, favicon_base_64: str) -> None:\n        from urllib.parse import urlparse\n        import base64\n        import binascii\n        decoded = None\n\n        if url:\n            try:\n                parsed = urlparse(url)\n                filename = os.path.basename(parsed.path)\n                (base, extension) = filename.lower().strip().rsplit('.', 1)\n            except ValueError:\n                logger.error(f\"UUID: {self.get('uuid')} Cant work out file extension from '{url}'\")\n                return None\n        else:\n            # Assume favicon.ico\n            base = \"favicon\"\n            extension = \"ico\"\n\n        fname = os.path.join(self.data_dir, f\"favicon.{extension}\")\n\n        try:\n            # validate=True makes sure the string only contains valid base64 chars\n            decoded = base64.b64decode(favicon_base_64, validate=True)\n        except (binascii.Error, ValueError) as e:\n            logger.warning(f\"UUID: {self.get('uuid')} FavIcon save data (Base64) corrupt? {str(e)}\")\n        else:\n            if decoded:\n                try:\n                    with open(fname, 'wb') as f:\n                        f.write(decoded)\n\n                    # Invalidate module-level favicon filename cache for this watch\n                    _FAVICON_FILENAME_CACHE.pop(self.data_dir, None)\n\n                    # A signal that could trigger the socket server to update the browser also\n                    watch_check_update = signal('watch_favicon_bump')\n                    if watch_check_update:\n                        watch_check_update.send(watch_uuid=self.get('uuid'))\n\n                except Exception as e:\n                    logger.warning(f\"UUID: {self.get('uuid')} error saving FavIcon to {fname} - {str(e)}\")\n\n        # @todo - Store some checksum and only write when its different\n        logger.debug(f\"UUID: {self.get('uuid')} updated favicon to at {fname}\")\n\n    def get_favicon_filename(self) -> str | None:\n        \"\"\"\n        Find any favicon.* file in the watch data directory.\n\n        Uses a module-level cache keyed by data_dir to survive Watch object recreation,\n        deepcopy (which drops instance attrs), and concurrent request races.\n        Invalidated by bump_favicon() when a new favicon is saved.\n\n        Returns:\n            str: Basename of the favicon file, or None if not found.\n        \"\"\"\n        if self.data_dir in _FAVICON_FILENAME_CACHE:\n            return _FAVICON_FILENAME_CACHE[self.data_dir]\n\n        import glob\n        files = glob.glob(os.path.join(self.data_dir, \"favicon.*\"))\n        fname = os.path.basename(files[0]) if files else None\n        _FAVICON_FILENAME_CACHE[self.data_dir] = fname\n        return fname\n\n    def get_screenshot_as_thumbnail(self, max_age=3200):\n        \"\"\"Return path to a square thumbnail of the most recent screenshot.\n\n        Creates a 150x150 pixel thumbnail from the top portion of the screenshot.\n\n        Args:\n            max_age: Maximum age in seconds before recreating thumbnail\n\n        Returns:\n            Path to thumbnail or None if no screenshot exists\n        \"\"\"\n        import os\n        import time\n\n        thumbnail_path = os.path.join(self.data_dir, \"thumbnail.jpeg\")\n        top_trim = 500  # Pixels from top of screenshot to use\n\n        screenshot_path = self.get_screenshot()\n        if not screenshot_path:\n            return None\n\n        # Reuse thumbnail if it's fresh and screenshot hasn't changed\n        if os.path.isfile(thumbnail_path):\n            thumbnail_mtime = os.path.getmtime(thumbnail_path)\n            screenshot_mtime = os.path.getmtime(screenshot_path)\n\n            if screenshot_mtime <= thumbnail_mtime and time.time() - thumbnail_mtime < max_age:\n                return thumbnail_path\n\n        try:\n            from PIL import Image\n\n            with Image.open(screenshot_path) as img:\n                # Crop top portion first (full width, top_trim height)\n                top_crop_height = min(top_trim, img.height)\n                img = img.crop((0, 0, img.width, top_crop_height))\n\n                # Create a smaller intermediate image (to reduce memory usage)\n                aspect = img.width / img.height\n                interim_width = min(top_trim, img.width)\n                interim_height = int(interim_width / aspect) if aspect > 0 else top_trim\n                img = img.resize((interim_width, interim_height), Image.NEAREST)\n\n                # Convert to RGB if needed\n                if img.mode != 'RGB':\n                    img = img.convert('RGB')\n\n                # Crop to square from top center\n                square_size = min(img.width, img.height)\n                left = (img.width - square_size) // 2\n                img = img.crop((left, 0, left + square_size, square_size))\n\n                # Final resize to exact thumbnail size with better filter\n                img = img.resize((350, 350), Image.BILINEAR)\n\n                # Save with optimized settings\n                img.save(thumbnail_path, \"JPEG\", quality=75, optimize=True)\n\n            return thumbnail_path\n\n        except Exception as e:\n            logger.error(f\"Error creating thumbnail for {self.get('uuid')}: {str(e)}\")\n            return None\n\n    def __get_file_ctime(self, filename):\n        fname = os.path.join(self.data_dir, filename)\n        if os.path.isfile(fname):\n            return int(os.path.getmtime(fname))\n        return False\n\n    @property\n    def error_text_ctime(self):\n        return self.__get_file_ctime('last-error.txt')\n\n    @property\n    def snapshot_text_ctime(self):\n        if self.history_n==0:\n            return False\n\n        timestamp = list(self.history.keys())[-1]\n        return int(timestamp)\n\n    @property\n    def snapshot_screenshot_ctime(self):\n        return self.__get_file_ctime('last-screenshot.png')\n\n    @property\n    def snapshot_error_screenshot_ctime(self):\n        return self.__get_file_ctime('last-error-screenshot.png')\n\n    def get_error_text(self):\n        \"\"\"Return the text saved from a previous request that resulted in a non-200 error\"\"\"\n        fname = os.path.join(self.data_dir, \"last-error.txt\")\n        if os.path.isfile(fname):\n            with open(fname, 'r', encoding='utf-8') as f:\n                return f.read()\n        return False\n\n    def get_error_snapshot(self):\n        \"\"\"Return path to the screenshot that resulted in a non-200 error\"\"\"\n        fname = os.path.join(self.data_dir, \"last-error-screenshot.png\")\n        if os.path.isfile(fname):\n            return fname\n        return False\n\n\n    def pause(self):\n        self['paused'] = True\n\n    def unpause(self):\n        self['paused'] = False\n\n    def toggle_pause(self):\n        self['paused'] ^= True\n\n    def mute(self):\n        self['notification_muted'] = True\n\n    def unmute(self):\n        self['notification_muted'] = False\n\n    def toggle_mute(self):\n        self['notification_muted'] ^= True\n\n    def _get_commit_data(self):\n        \"\"\"\n        Prepare watch data for commit.\n\n        Excludes processor_config_* keys (stored in separate files).\n        Normalizes browser_steps to empty list if no meaningful steps.\n        \"\"\"\n        import copy\n\n        # Get base snapshot with lock\n        lock = self._datastore.lock if self._datastore and hasattr(self._datastore, 'lock') else None\n\n        if lock:\n            with lock:\n                snapshot = dict(self)\n        else:\n            snapshot = dict(self)\n\n        # Exclude processor config keys (stored separately)\n        watch_dict = {k: copy.deepcopy(v) for k, v in snapshot.items() if not k.startswith('processor_config_')}\n\n        # Normalize browser_steps: if no meaningful steps, save as empty list\n        if not self.has_browser_steps:\n            watch_dict['browser_steps'] = []\n\n        return watch_dict\n\n    # _save_to_disk() method provided by EntityPersistenceMixin\n    # commit() method inherited from watch_base\n\n\n    def extra_notification_token_values(self):\n        # Used for providing extra tokens\n        # return {'widget': 555}\n        return {}\n\n    def extra_notification_token_placeholder_info(self):\n        # Used for providing extra tokens\n        # return [('widget', \"Get widget amounts\")]\n        return []\n\n\n    def extract_regex_from_all_history(self, regex):\n        import csv\n        import re\n        import datetime\n        csv_output_filename = False\n        csv_writer = False\n        f = None\n\n        # self.history will be keyed with the full path\n        for k, fname in self.history.items():\n            if os.path.isfile(fname):\n                if True:\n                    contents = self.get_history_snapshot(timestamp=k)\n                    res = re.findall(regex, contents, re.MULTILINE)\n                    if res:\n                        if not csv_writer:\n                            # A file on the disk can be transferred much faster via flask than a string reply\n                            csv_output_filename = f\"report-{self.get('uuid')}.csv\"\n                            f = open(os.path.join(self.data_dir, csv_output_filename), 'w')\n                            # @todo some headers in the future\n                            #fieldnames = ['Epoch seconds', 'Date']\n                            csv_writer = csv.writer(f,\n                                                    delimiter=',',\n                                                    quotechar='\"',\n                                                    quoting=csv.QUOTE_MINIMAL,\n                                                    #fieldnames=fieldnames\n                                                    )\n                            csv_writer.writerow(['Epoch seconds', 'Date'])\n                            # csv_writer.writeheader()\n\n                        date_str = datetime.datetime.fromtimestamp(int(k)).strftime('%Y-%m-%d %H:%M:%S')\n                        for r in res:\n                            row = [k, date_str]\n                            if isinstance(r, str):\n                                row.append(r)\n                            else:\n                                row+=r\n                            csv_writer.writerow(row)\n\n        if f:\n            f.close()\n\n        return csv_output_filename\n\n\n    def has_special_diff_filter_options_set(self):\n\n        # All False - nothing would be done, so act like it's not processable\n        if not self.get('filter_text_added', True) and not self.get('filter_text_replaced', True) and not self.get('filter_text_removed', True):\n            return False\n\n        # Or one is set\n        if not self.get('filter_text_added', True) or not self.get('filter_text_replaced', True) or not self.get('filter_text_removed', True):\n            return True\n\n        # None is set\n        return False\n\n    def save_error_text(self, contents):\n        self.ensure_data_dir_exists()\n        target_path = os.path.join(self.data_dir, \"last-error.txt\")\n        with open(target_path, 'w', encoding='utf-8') as f:\n            f.write(contents)\n\n    def save_xpath_data(self, data, as_error=False):\n        import json\n        import zlib\n\n        if as_error:\n            target_path = os.path.join(str(self.data_dir), \"elements-error.deflate\")\n        else:\n            target_path = os.path.join(str(self.data_dir), \"elements.deflate\")\n\n        self.ensure_data_dir_exists()\n\n        with open(target_path, 'wb') as f:\n            if not isinstance(data, str):\n                f.write(zlib.compress(json.dumps(data).encode()))\n            else:\n                f.write(zlib.compress(data.encode()))\n            f.close()\n\n    # Save as PNG, PNG is larger but better for doing visual diff in the future\n    def save_screenshot(self, screenshot: bytes, as_error=False):\n\n        if as_error:\n            target_path = os.path.join(self.data_dir, \"last-error-screenshot.png\")\n        else:\n            target_path = os.path.join(self.data_dir, \"last-screenshot.png\")\n\n        self.ensure_data_dir_exists()\n\n        with open(target_path, 'wb') as f:\n            f.write(screenshot)\n            f.close()\n\n\n    def get_last_fetched_text_before_filters(self):\n        import brotli\n        filepath = os.path.join(self.data_dir, 'last-fetched.br')\n\n        if not os.path.isfile(filepath) or os.path.getsize(filepath) == 0:\n            # If a previous attempt doesnt yet exist, just snarf the previous snapshot instead\n            dates = list(self.history.keys())\n            if len(dates):\n                return self.get_history_snapshot(timestamp=dates[-1])\n            else:\n                return ''\n\n        with open(filepath, 'rb') as f:\n            return(brotli.decompress(f.read()).decode('utf-8'))\n\n    def save_last_text_fetched_before_filters(self, contents):\n        import brotli\n        filepath = os.path.join(self.data_dir, 'last-fetched.br')\n        _brotli_save(contents, filepath, mode=brotli.MODE_TEXT, fallback_uncompressed=False)\n\n    def save_last_fetched_html(self, timestamp, contents):\n        self.ensure_data_dir_exists()\n        snapshot_fname = f\"{timestamp}.html.br\"\n        filepath = os.path.join(self.data_dir, snapshot_fname)\n        _brotli_save(contents, filepath, mode=None, fallback_uncompressed=True)\n        self._prune_last_fetched_html_snapshots()\n\n    def get_fetched_html(self, timestamp):\n        import brotli\n\n        snapshot_fname = f\"{timestamp}.html.br\"\n        filepath = os.path.join(self.data_dir, snapshot_fname)\n        if os.path.isfile(filepath):\n            with open(filepath, 'rb') as f:\n                return (brotli.decompress(f.read()).decode('utf-8'))\n\n        return False\n\n\n    def _prune_last_fetched_html_snapshots(self):\n\n        dates = list(self.history.keys())\n        dates.reverse()\n\n        for index, timestamp in enumerate(dates):\n            snapshot_fname = f\"{timestamp}.html.br\"\n            filepath = os.path.join(self.data_dir, snapshot_fname)\n\n            # Keep only the first 2\n            if index > 1 and os.path.isfile(filepath):\n                os.remove(filepath)\n\n\n    @property\n    def get_browsersteps_available_screenshots(self):\n        \"For knowing which screenshots are available to show the user in BrowserSteps UI\"\n        available = []\n        for f in Path(self.data_dir).glob('step_before-*.jpeg'):\n            step_n=re.search(r'step_before-(\\d+)', f.name)\n            if step_n:\n                available.append(step_n.group(1))\n        return available\n\n    def compile_error_texts(self, has_proxies=None):\n        \"\"\"Compile error texts for this watch.\n        Accepts has_proxies parameter to ensure it works even outside app context\"\"\"\n        from flask import url_for, has_request_context\n        from markupsafe import Markup\n\n        output = []  # Initialize as list since we're using append\n        last_error = self.get('last_error','')\n\n        has_app_context = has_request_context()\n\n        # has app+request context, we can use url_for()\n        if has_app_context:\n            if last_error:\n                last_error = safe_jinja.render_fully_escaped(last_error)\n                if '403' in last_error:\n                    if has_proxies:\n                        output.append(str(Markup(f\"{last_error} - Try other proxies/location '\")))\n                    else:\n                        output.append(str(Markup(f\"{last_error} - Try adding external proxies/locations '\")))\n                else:\n                    output.append(str(Markup(last_error)))\n\n            if self.get('last_notification_error'):\n                txt = safe_jinja.render_fully_escaped(self.get('last_notification_error'))\n                result = f'
{txt}
'\n                output.append(result)\n\n        else:\n            # Lo_Fi version - no app context, cant rely on Jinja2 Markup\n            if last_error:\n                output.append(safe_jinja.render_fully_escaped(last_error))\n            if self.get('last_notification_error'):\n                output.append(safe_jinja.render_fully_escaped(self.get('last_notification_error')))\n\n        res = \"\\n\".join(output)\n        return res\n\n"
  },
  {
    "path": "changedetectionio/model/__init__.py",
    "content": "import os\nimport uuid\n\nfrom changedetectionio import strtobool\nfrom .persistence import EntityPersistenceMixin, _determine_entity_type\n\n__all__ = ['EntityPersistenceMixin', 'watch_base']\n\nfrom ..browser_steps.browser_steps import browser_steps_get_valid_steps\n\nUSE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH = 'System default'\nCONDITIONS_MATCH_LOGIC_DEFAULT = 'ALL'\n\n\nclass watch_base(dict):\n    \"\"\"\n    Base watch domain model (inherits from dict for backward compatibility).\n\n    WARNING: This class inherits from dict, which violates proper encapsulation.\n    Dict inheritance is legacy technical debt that should be refactored to a proper\n    domain model (e.g., Pydantic BaseModel) for better type safety and validation.\n\n    TODO: Migrate to Pydantic BaseModel for:\n          - Type safety and IDE autocomplete\n          - Automatic validation\n          - Clear separation between domain model and serialization\n          - Database backend abstraction (file → postgres → mongodb)\n          - Configuration override chain resolution (Watch → Tag → Global)\n          - Immutability options\n          - Better testing\n          - USE https://docs.pydantic.dev/latest/integrations/datamodel_code_generator TO BUILD THE MODEL FROM THE API-SPEC!!!\n\n    CHAIN RESOLUTION ARCHITECTURE:\n        The dream is a 3-level override hierarchy:\n            Watch settings → Tag/Group settings → Global settings\n\n        Current implementation: MANUAL resolution scattered across codebase\n        - Processors manually check watch.get('field')\n        - Loop through tags to find overrides_watch=True\n        - Fall back to datastore['settings']['application']['field']\n\n        Pydantic implementation: AUTOMATIC resolution via @computed_field\n        - Single source of truth for each setting's resolution logic\n        - Type-safe, testable, self-documenting\n        - Example: watch.resolved_fetch_backend (instead of nested dict navigation)\n\n        See: Watch.py model docstring for detailed Pydantic architecture plan\n        See: Tag.py model docstring for tag override explanation\n        See: processors/restock_diff/processor.py:184-192 for current manual example\n\n    Core Fields:\n        uuid (str): Unique identifier for this watch (auto-generated)\n        url (str): Target URL to monitor for changes\n        title (str|None): Custom display name (overrides page_title if set)\n        page_title (str|None): Title extracted from  tag of monitored page\n        tags (List[str]): List of tag UUIDs for categorization\n        tag (str): DEPRECATED - Old single-tag system, use tags instead\n\n    Check Configuration:\n        processor (str): Processor type ('text_json_diff', 'restock_diff', etc.)\n        fetch_backend (str): Fetcher to use ('system', 'html_requests', 'playwright', etc.)\n        method (str): HTTP method ('GET', 'POST', etc.)\n        headers (dict): Custom HTTP headers to send\n        proxy (str|None): Preferred proxy server\n        paused (bool): Whether change detection is paused\n\n    Scheduling:\n        time_between_check (dict): Check interval {'weeks': int, 'days': int, 'hours': int, 'minutes': int, 'seconds': int}\n        time_between_check_use_default (bool): Use global default interval if True\n        time_schedule_limit (dict): Weekly schedule limiting when checks can run\n            Structure: {\n                'enabled': bool,\n                'monday/tuesday/.../sunday': {\n                    'enabled': bool,\n                    'start_time': str ('HH:MM'),\n                    'duration': {'hours': str, 'minutes': str}\n                }\n            }\n\n    Content Filtering:\n        include_filters (List[str]): CSS/XPath selectors to extract content\n        subtractive_selectors (List[str]): Selectors to remove from content\n        ignore_text (List[str]): Text patterns to ignore in change detection\n        trigger_text (List[str]): Text/regex that must be present to trigger change\n        text_should_not_be_present (List[str]): Text that should NOT be present\n        extract_text (List[str]): Regex patterns to extract specific text after filtering\n\n    Text Processing:\n        trim_text_whitespace (bool): Strip leading/trailing whitespace\n        sort_text_alphabetically (bool): Sort lines alphabetically before comparison\n        remove_duplicate_lines (bool): Remove duplicate lines\n        check_unique_lines (bool): Compare against all history for unique lines\n        strip_ignored_lines (bool|None): Remove lines matching ignore patterns\n\n    Change Detection Filters:\n        filter_text_added (bool): Include added text in change detection\n        filter_text_removed (bool): Include removed text in change detection\n        filter_text_replaced (bool): Include replaced text in change detection\n\n    Browser Automation:\n        browser_steps (List[dict]): Browser automation steps for JS-heavy sites\n        browser_steps_last_error_step (int|None): Last step that caused error\n        webdriver_delay (int|None): Seconds to wait after page load\n        webdriver_js_execute_code (str|None): JavaScript to execute before extraction\n\n    Restock Detection:\n        in_stock_only (bool): Only trigger on in-stock transitions\n        follow_price_changes (bool): Monitor price changes\n        has_ldjson_price_data (bool|None): Whether page has LD-JSON price data\n        track_ldjson_price_data (str|None): Track LD-JSON price data ('ACCEPT', 'REJECT', None)\n        price_change_threshold_percent (float|None): Minimum price change % to trigger\n\n    Notifications:\n        notification_urls (List[str]): Apprise URLs for notifications\n        notification_title (str|None): Custom notification title template\n        notification_body (str|None): Custom notification body template\n        notification_format (str): Notification format (e.g., 'System default', 'Text', 'HTML')\n        notification_muted (bool): Disable notifications for this watch\n        notification_screenshot (bool): Include screenshot in notifications\n        notification_alert_count (int): Number of notifications sent\n        last_notification_error (str|None): Last notification error message\n        body (str|None): DEPRECATED? Legacy notification body field\n        filter_failure_notification_send (bool): Send notification on filter failures\n\n    History & State:\n        date_created (int|None): Unix timestamp of watch creation\n        last_checked (int): Unix timestamp of last check\n        last_viewed (int): History snapshot key of last user view\n        last_error (str|bool): Last error message or False if no error\n        check_count (int): Total number of checks performed\n        fetch_time (float): Duration of last fetch in seconds\n        consecutive_filter_failures (int): Counter for consecutive filter match failures\n        previous_md5 (str|bool): MD5 hash of previous content\n        history_snapshot_max_length (int|None): Max history snapshots to keep (None = use global)\n\n    Conditions:\n        conditions (dict): Custom conditions for change detection logic\n        conditions_match_logic (str): Logic operator ('ALL', 'ANY') for conditions\n\n    Metadata:\n        content-type (str|None): Content-Type from last fetch\n        remote_server_reply (str|None): Server header from last response\n        ignore_status_codes (List[int]|None): HTTP status codes to ignore\n        use_page_title_in_list (bool|None): Display page title in watch list (None = use system default)\n\n    Instance Attributes (not serialized):\n        __datastore: Reference to parent DataStore (set externally after creation)\n        data_dir: Filesystem path for this watch's data directory\n\n    Notes:\n        - Many fields default to None to distinguish \"not set\" from \"set to default\"\n        - When field is None, system-level defaults are used\n        - Processor-specific configs (e.g., processor_config_*) are NOT stored in watch.json\n          They are stored in separate {processor_name}.json files\n        - This class is used for both Watch and Tag objects (tags reuse the structure)\n    \"\"\"\n\n    def __init__(self, *arg, **kw):\n        # Store datastore reference (common to Watch and Tag)\n        # Use single underscore to avoid name mangling issues in subclasses\n        self._datastore = kw.get('__datastore')\n        if kw.get('__datastore'):\n            del kw['__datastore']\n\n        # Store datastore_path (common to Watch and Tag)\n        self._datastore_path = kw.get('datastore_path')\n        if kw.get('datastore_path'):\n            del kw['datastore_path']\n\n        # IMPORTANT: Don't initialize __watch_was_edited yet!\n        # We'll initialize it AFTER the initial update() call below\n        # This prevents marking the watch as edited during initialization\n\n        self.update({\n            # Custom notification content\n            # Re #110, so then if this is set to None, we know to use the default value instead\n            # Requires setting to None on submit if it's the same as the default\n            # Should be all None by default, so we use the system default in this case.\n            'body': None,\n            'browser_steps': [],\n            'browser_steps_last_error_step': None,\n            'conditions' : [],\n            'conditions_match_logic': CONDITIONS_MATCH_LOGIC_DEFAULT,\n            'check_count': 0,\n            'check_unique_lines': False,  # On change-detected, compare against all history if its something new\n            'consecutive_filter_failures': 0,  # Every time the CSS/xPath filter cannot be located, reset when all is fine.\n            'content-type': None,\n            'date_created': None,\n            'extract_text': [],  # Extract text by regex after filters\n            'fetch_backend': 'system',  # plaintext, playwright etc\n            'fetch_time': 0.0,\n            'filter_failure_notification_send': strtobool(os.getenv('FILTER_FAILURE_NOTIFICATION_SEND_DEFAULT', 'True')),\n            'filter_text_added': True,\n            'filter_text_removed': True,\n            'filter_text_replaced': True,\n            'follow_price_changes': True,\n            'has_ldjson_price_data': None,\n            'history_snapshot_max_length': None,\n            'headers': {},  # Extra headers to send\n            'ignore_text': [],  # List of text to ignore when calculating the comparison checksum\n            'ignore_status_codes': None,\n            'in_stock_only': True,  # Only trigger change on going to instock from out-of-stock\n            'include_filters': [],\n            'last_checked': 0,\n            'last_error': False,\n            'last_notification_error': None,\n            'last_viewed': 0,  # history key value of the last viewed via the [diff] link\n            'method': 'GET',\n            'notification_alert_count': 0,\n            'notification_body': None,\n            'notification_format': USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH,\n            'notification_muted': False,\n            'notification_screenshot': False,  # Include the latest screenshot if available and supported by the apprise URL\n            'notification_title': None,\n            'notification_urls': [],  # List of URLs to add to the notification Queue (Usually AppRise)\n            'page_title': None, # <title> from the page\n            'paused': False,\n            'previous_md5': False,\n            'processor': 'text_json_diff',  # could be restock_diff or others from .processors\n            'price_change_threshold_percent': None,\n            'proxy': None,  # Preferred proxy connection\n            'remote_server_reply': None,  # From 'server' reply header\n            'sort_text_alphabetically': False,\n            'strip_ignored_lines': None,\n            'subtractive_selectors': [],\n            'tag': '',  # Old system of text name for a tag, to be removed\n            'tags': [],  # list of UUIDs to App.Tags\n            'text_should_not_be_present': [],  # Text that should not present\n            'time_between_check': {'weeks': None, 'days': None, 'hours': None, 'minutes': None, 'seconds': None},\n            'time_between_check_use_default': True,\n            \"time_schedule_limit\": {\n                \"enabled\": False,\n                \"monday\": {\n                    \"enabled\": True,\n                    \"start_time\": \"00:00\",\n                    \"duration\": {\n                        \"hours\": \"24\",\n                        \"minutes\": \"00\"\n                    }\n                },\n                \"tuesday\": {\n                    \"enabled\": True,\n                    \"start_time\": \"00:00\",\n                    \"duration\": {\n                        \"hours\": \"24\",\n                        \"minutes\": \"00\"\n                    }\n                },\n                \"wednesday\": {\n                    \"enabled\": True,\n                    \"start_time\": \"00:00\",\n                    \"duration\": {\n                        \"hours\": \"24\",\n                        \"minutes\": \"00\"\n                    }\n                },\n                \"thursday\": {\n                    \"enabled\": True,\n                    \"start_time\": \"00:00\",\n                    \"duration\": {\n                        \"hours\": \"24\",\n                        \"minutes\": \"00\"\n                    }\n                },\n                \"friday\": {\n                    \"enabled\": True,\n                    \"start_time\": \"00:00\",\n                    \"duration\": {\n                        \"hours\": \"24\",\n                        \"minutes\": \"00\"\n                    }\n                },\n                \"saturday\": {\n                    \"enabled\": True,\n                    \"start_time\": \"00:00\",\n                    \"duration\": {\n                        \"hours\": \"24\",\n                        \"minutes\": \"00\"\n                    }\n                },\n                \"sunday\": {\n                    \"enabled\": True,\n                    \"start_time\": \"00:00\",\n                    \"duration\": {\n                        \"hours\": \"24\",\n                        \"minutes\": \"00\"\n                    }\n                },\n            },\n            'title': None, # An arbitrary field that overrides 'page_title'\n            'track_ldjson_price_data': None,\n            'trim_text_whitespace': False,\n            'remove_duplicate_lines': False,\n            'trigger_text': [],  # List of text or regex to wait for until a change is detected\n            'url': '',\n            'use_page_title_in_list': None, # None = use system settings\n            'uuid': str(uuid.uuid4()),\n            'webdriver_delay': None,\n            'webdriver_js_execute_code': None,  # Run before change-detection\n        })\n\n        super(watch_base, self).__init__(*arg, **kw)\n\n        # Check if we're being initialized from an existing watch object\n        # that has was_edited=True, so we can preserve the flag\n        preserve_edited_flag = False\n        if self.get('default'):\n            # When creating a new watch object from an existing one (e.g., changing processor),\n            # preserve the was_edited flag if it was True\n            default_watch = self.get('default')\n            if hasattr(default_watch, 'was_edited') and default_watch.was_edited:\n                preserve_edited_flag = True\n            del self['default']\n\n        # NOW initialize the edited flag after all initial setup is complete\n        # This ensures initialization doesn't trigger the edited flag\n        # But preserve it if the source watch had it set to True\n        self.__watch_was_edited = preserve_edited_flag\n\n    def _mark_field_as_edited(self, key):\n        \"\"\"\n        Helper to mark a field as edited if it's writable.\n\n        Internal method used by __setitem__, update(), pop(), etc.\n        \"\"\"\n        # Don't track edits during initial load or if already edited\n        if not hasattr(self, '_watch_base__watch_was_edited'):\n            return\n        if self.__watch_was_edited:\n            return  # Already marked as edited\n\n        # Import from shared schema utilities (no circular dependency)\n        from .schema_utils import get_readonly_watch_fields\n        readonly_fields = get_readonly_watch_fields()\n\n        # Additional system-managed fields not in OpenAPI spec (yet)\n        # These are set by processors/workers and should not trigger edited flag\n        additional_system_fields = {\n            'last_check_status',  # Set by processors\n            'restock',  # Set by restock processor\n            'last_viewed',  # Set by mark_all_viewed endpoint\n        }\n\n        # Only mark as edited if this is a user-writable field\n        if key not in readonly_fields and key not in additional_system_fields:\n            self.__watch_was_edited = True\n\n    def __setitem__(self, key, value):\n        \"\"\"\n        Override dict.__setitem__ to track when writable watch fields are modified.\n\n        This enables skipping reprocessing when:\n        1. HTML content is unchanged (checksumFromPreviousCheckWasTheSame)\n        2. AND watch configuration was not edited\n\n        Only sets the edited flag when field is NOT in readonly_fields (from OpenAPI spec).\n        \"\"\"\n        # Set the value first (always)\n        super().__setitem__(key, value)\n        # Mark as edited if writable field\n        self._mark_field_as_edited(key)\n\n    def __delitem__(self, key):\n        \"\"\"Override dict.__delitem__ to track deletions of writable fields.\"\"\"\n        super().__delitem__(key)\n        self._mark_field_as_edited(key)\n\n    def update(self, *args, **kwargs):\n\n        if args and args[0].get('browser_steps'):\n            args[0]['browser_steps'] = browser_steps_get_valid_steps(args[0].get('browser_steps'))\n\n        \"\"\"Override dict.update() to track modifications to writable fields.\"\"\"\n        # Call parent update first\n        super().update(*args, **kwargs)\n\n        # Mark as edited for any writable fields that were updated\n        # Handle both update(dict) and update(key=value) forms\n        if args:\n            for key in args[0].keys():\n                self._mark_field_as_edited(key)\n        for key in kwargs.keys():\n            self._mark_field_as_edited(key)\n\n\n    def pop(self, key, *args):\n        \"\"\"Override dict.pop() to track removal of writable fields.\"\"\"\n        result = super().pop(key, *args)\n        self._mark_field_as_edited(key)\n        return result\n\n    def setdefault(self, key, default=None):\n        \"\"\"Override dict.setdefault() to track modifications to writable fields.\"\"\"\n        # Only marks as edited if key didn't exist (i.e., a new value was set)\n        existed = key in self\n        result = super().setdefault(key, default)\n        if not existed:\n            self._mark_field_as_edited(key)\n        return result\n\n    @property\n    def was_edited(self):\n        \"\"\"\n        Check if watch configuration was edited since last processing.\n\n        Returns:\n            bool: True if writable fields were modified, False otherwise\n        \"\"\"\n        return getattr(self, '_watch_base__watch_was_edited', False)\n\n    def reset_watch_edited_flag(self):\n        \"\"\"\n        Reset the watch edited flag after successful processing.\n\n        Call this after processing completes to allow future content-only change detection.\n        \"\"\"\n        self.__watch_was_edited = False\n\n    @classmethod\n    def get_property_names(cls):\n        \"\"\"\n        Get all @property attribute names from this model class using introspection.\n\n        This discovers computed/derived properties that are not stored in the datastore.\n        These properties should be filtered out during PUT/POST requests.\n\n        Returns:\n            frozenset: Immutable set of @property attribute names from the model class\n        \"\"\"\n        import functools\n\n        # Create a cached version if it doesn't exist\n        if not hasattr(cls, '_cached_get_property_names'):\n            @functools.cache\n            def _get_props():\n                properties = set()\n                # Use introspection to find all @property attributes\n                for name in dir(cls):\n                    # Skip private/magic attributes\n                    if name.startswith('_'):\n                        continue\n                    try:\n                        attr = getattr(cls, name)\n                        # Check if it's a property descriptor\n                        if isinstance(attr, property):\n                            properties.add(name)\n                    except (AttributeError, TypeError):\n                        continue\n                return frozenset(properties)\n\n            cls._cached_get_property_names = _get_props\n\n        return cls._cached_get_property_names()\n\n    def __deepcopy__(self, memo):\n        \"\"\"\n        Custom deepcopy for all watch_base subclasses (Watch, Tag, etc.).\n\n        CRITICAL FIX: Prevents copying large reference objects like __datastore\n        which would cause exponential memory growth when Watch objects are deepcopied.\n\n        This is called by:\n        - api/Watch.py:76 (API endpoint)\n        - api/Tags.py:28 (Tags API)\n        - processors/base.py:26 (EVERY processor run)\n        - store/__init__.py:544 (clone watch)\n        - And other locations\n        \"\"\"\n        from copy import deepcopy\n\n        # Create new instance without calling __init__\n        cls = self.__class__\n        new_obj = cls.__new__(cls)\n        memo[id(self)] = new_obj\n\n        # Copy the dict data (all the settings)\n        for key, value in self.items():\n            new_obj[key] = deepcopy(value, memo)\n\n        # Copy instance attributes dynamically\n        # This handles Watch-specific attrs (like __datastore) and any future subclass attrs\n        for attr_name in dir(self):\n            # Skip methods, special attrs, and dict keys\n            if attr_name.startswith('_') and not attr_name.startswith('__'):\n                # This catches _model__datastore, _model__history_n, etc.\n                try:\n                    attr_value = getattr(self, attr_name)\n\n                    # Special handling: Share references to large objects instead of copying\n                    # Examples: _datastore, __datastore, __app_reference, __global_settings, etc.\n                    if (attr_name == '_datastore' or\n                        attr_name.endswith('__datastore') or\n                        attr_name.endswith('__app')):\n                        # Share the reference (don't copy!) to prevent memory leaks\n                        setattr(new_obj, attr_name, attr_value)\n                    # Skip cache attributes - let them regenerate on demand\n                    elif 'cache' in attr_name.lower():\n                        pass  # Don't copy caches\n                    # Copy regular instance attributes\n                    elif not callable(attr_value):\n                        setattr(new_obj, attr_name, attr_value)\n                except AttributeError:\n                    pass  # Attribute doesn't exist in this instance\n\n        return new_obj\n\n    def __getstate__(self):\n        \"\"\"\n        Custom pickle serialization for all watch_base subclasses.\n\n        Excludes large reference objects (like __datastore) from serialization.\n        \"\"\"\n        # Get the dict data\n        state = dict(self)\n\n        # Collect instance attributes (excluding methods and large references)\n        instance_attrs = {}\n        for attr_name in dir(self):\n            if attr_name.startswith('_') and not attr_name.startswith('__'):\n                try:\n                    attr_value = getattr(self, attr_name)\n                    # Exclude large reference objects and caches from serialization\n                    if not (attr_name == '_datastore' or\n                           attr_name.endswith('__datastore') or\n                           attr_name.endswith('__app') or\n                           'cache' in attr_name.lower() or\n                           callable(attr_value)):\n                        instance_attrs[attr_name] = attr_value\n                except AttributeError:\n                    pass\n\n        if instance_attrs:\n            state['__instance_metadata__'] = instance_attrs\n\n        return state\n\n    def __setstate__(self, state):\n        \"\"\"\n        Custom pickle deserialization for all watch_base subclasses.\n\n        WARNING: Large reference objects (like __datastore) are NOT restored!\n        Caller must restore these references after unpickling if needed.\n        \"\"\"\n        # Extract metadata\n        metadata = state.pop('__instance_metadata__', {})\n\n        # Restore dict data\n        self.update(state)\n\n        # Restore instance attributes\n        for attr_name, attr_value in metadata.items():\n            setattr(self, attr_name, attr_value)\n\n    @property\n    def data_dir(self):\n        \"\"\"\n        The base directory for this watch/tag data (property, computed from UUID).\n\n        Common property for both Watch and Tag objects.\n        Returns path like: /datastore/{uuid}/\n        \"\"\"\n        return os.path.join(self._datastore_path, self['uuid']) if self._datastore_path else None\n\n    def ensure_data_dir_exists(self):\n        \"\"\"\n        Create the data directory if it doesn't exist.\n\n        Common method for both Watch and Tag objects.\n        \"\"\"\n        from loguru import logger\n        if not os.path.isdir(self.data_dir):\n            logger.debug(f\"> Creating data dir {self.data_dir}\")\n            os.mkdir(self.data_dir)\n\n    def get_global_setting(self, *path):\n        \"\"\"\n        Get a setting from the global datastore configuration.\n\n        Args:\n            *path: Path to the setting (e.g., 'application', 'history_snapshot_max_length')\n\n        Returns:\n            The setting value, or None if not found\n\n        Example:\n            maxlen = self.get_global_setting('application', 'history_snapshot_max_length')\n        \"\"\"\n        if not self._datastore:\n            return None\n\n        try:\n            value = self._datastore['settings']\n            for key in path:\n                value = value[key]\n            return value\n        except (KeyError, TypeError):\n            return None\n\n    def _get_commit_data(self):\n        \"\"\"\n        Prepare data for commit (can be overridden by subclasses).\n\n        Returns:\n            dict: Data to serialize (filtered as needed by subclass)\n        \"\"\"\n        import copy\n\n        # Acquire datastore lock to prevent concurrent modifications during copy\n        lock = self._datastore.lock if self._datastore and hasattr(self._datastore, 'lock') else None\n\n        if lock:\n            with lock:\n                snapshot = dict(self)\n        else:\n            snapshot = dict(self)\n\n        # Deep copy snapshot (slower, but done outside lock to minimize contention)\n        # Subclasses can override to filter keys (e.g., Watch excludes processor_config_*)\n        return {k: copy.deepcopy(v) for k, v in snapshot.items()}\n\n    def _save_to_disk(self, data_dict, uuid):\n        \"\"\"\n        Save data to disk (must be implemented by subclasses).\n\n        Args:\n            data_dict: Dictionary to save\n            uuid: UUID for logging\n\n        Raises:\n            NotImplementedError: If subclass doesn't implement\n        \"\"\"\n        raise NotImplementedError(\"Subclass must implement _save_to_disk()\")\n\n    def commit(self):\n        \"\"\"\n        Save this watch/tag immediately to disk using atomic write.\n\n        Common commit logic for Watch and Tag objects.\n        Subclasses override _get_commit_data() and _save_to_disk() for specifics.\n\n        Fire-and-forget: Logs errors but does not raise exceptions.\n        Data remains in memory even if save fails, so next commit will retry.\n        \"\"\"\n        from loguru import logger\n\n        if not self.data_dir:\n            entity_type = self.__class__.__name__\n            logger.error(f\"Cannot commit {entity_type} {self.get('uuid')} without datastore_path\")\n            return\n\n        uuid = self.get('uuid')\n        if not uuid:\n            entity_type = self.__class__.__name__\n            logger.error(f\"Cannot commit {entity_type} without UUID\")\n            return\n\n        # Get data from subclass (may filter keys)\n        try:\n            data_dict = self._get_commit_data()\n        except Exception as e:\n            logger.error(f\"Failed to prepare commit data for {uuid}: {e}\")\n            return\n\n        # Save to disk via subclass implementation\n        try:\n            # Determine entity type from module name (Watch.py -> watch, Tag.py -> tag)\n            entity_type = _determine_entity_type(self.__class__)\n            filename = f\"{entity_type}.json\"\n            self._save_to_disk(data_dict, uuid)\n            logger.debug(f\"Committed {entity_type} {uuid} to {uuid}/{filename}\")\n        except Exception as e:\n            logger.error(f\"Failed to commit {uuid}: {e}\")"
  },
  {
    "path": "changedetectionio/model/persistence.py",
    "content": "\"\"\"\nEntity persistence mixin for Watch and Tag models.\n\nProvides file-based persistence using atomic writes.\n\"\"\"\n\nimport functools\nimport inspect\n\n\n@functools.lru_cache(maxsize=None)\ndef _determine_entity_type(cls):\n    \"\"\"\n    Determine entity type from class hierarchy (cached at class level).\n\n    Args:\n        cls: The class to inspect\n\n    Returns:\n        str: Entity type ('watch', 'tag', etc.)\n\n    Raises:\n        ValueError: If entity type cannot be determined\n    \"\"\"\n    for base_class in inspect.getmro(cls):\n        module_name = base_class.__module__\n        if module_name.startswith('changedetectionio.model.'):\n            # Get last part after dot: \"changedetectionio.model.Watch\" -> \"watch\"\n            return module_name.split('.')[-1].lower()\n\n    raise ValueError(\n        f\"Cannot determine entity type for {cls.__module__}.{cls.__name__}. \"\n        f\"Entity must inherit from a class in changedetectionio.model (Watch or Tag).\"\n    )\n\n\nclass EntityPersistenceMixin:\n    \"\"\"\n    Mixin providing file persistence for watch_base subclasses (Watch, Tag, etc.).\n\n    This mixin provides the _save_to_disk() method required by watch_base.commit().\n    It automatically determines the correct filename and size limits based on class hierarchy.\n\n    Usage:\n        class model(EntityPersistenceMixin, watch_base):  # in Watch.py\n            pass\n\n        class model(EntityPersistenceMixin, watch_base):  # in Tag.py\n            pass\n    \"\"\"\n\n    def _save_to_disk(self, data_dict, uuid):\n        \"\"\"\n        Save entity to disk using atomic write.\n\n        Implements the abstract method required by watch_base.commit().\n        Automatically determines filename and size limits from class hierarchy.\n\n        Args:\n            data_dict: Dictionary to save\n            uuid: UUID for logging\n\n        Raises:\n            ValueError: If entity type cannot be determined from class hierarchy\n        \"\"\"\n        # Import here to avoid circular dependency\n        from changedetectionio.store.file_saving_datastore import save_entity_atomic\n\n        # Determine entity type (cached at class level, not instance level)\n        entity_type = _determine_entity_type(self.__class__)\n\n        # Set filename and size limits based on entity type\n        filename = f'{entity_type}.json'\n        max_size_mb = 10 if entity_type == 'watch' else 1\n\n        # Save using generic function\n        save_entity_atomic(\n            self.data_dir,\n            uuid,\n            data_dict,\n            filename=filename,\n            entity_type=entity_type,\n            max_size_mb=max_size_mb\n        )\n"
  },
  {
    "path": "changedetectionio/model/schema_utils.py",
    "content": "\"\"\"\nSchema utilities for Watch and Tag models.\n\nProvides functions to extract readonly fields and properties from OpenAPI spec.\nShared by both the model layer and API layer to avoid circular dependencies.\n\"\"\"\n\nimport functools\n\n\n@functools.cache\ndef get_openapi_schema_dict():\n    \"\"\"\n    Get the raw OpenAPI spec dictionary for schema access.\n\n    Returns the YAML dict directly (not the OpenAPI object).\n    \"\"\"\n    import os\n    import yaml\n\n    spec_path = os.path.join(os.path.dirname(__file__), '../../docs/api-spec.yaml')\n    if not os.path.exists(spec_path):\n        spec_path = os.path.join(os.path.dirname(__file__), '../docs/api-spec.yaml')\n\n    with open(spec_path, 'r', encoding='utf-8') as f:\n        return yaml.safe_load(f)\n\n\n@functools.cache\ndef _resolve_readonly_fields(schema_name):\n    \"\"\"\n    Generic helper to resolve readOnly fields, including allOf inheritance.\n\n    Args:\n        schema_name: Name of the schema (e.g., 'Watch', 'Tag')\n\n    Returns:\n        frozenset: All readOnly field names including inherited ones\n    \"\"\"\n    spec_dict = get_openapi_schema_dict()\n    schema = spec_dict['components']['schemas'].get(schema_name, {})\n\n    readonly_fields = set()\n\n    # Handle allOf (schema inheritance)\n    if 'allOf' in schema:\n        for item in schema['allOf']:\n            # Resolve $ref to parent schema\n            if '$ref' in item:\n                ref_path = item['$ref'].split('/')[-1]\n                ref_schema = spec_dict['components']['schemas'].get(ref_path, {})\n                if 'properties' in ref_schema:\n                    for field_name, field_def in ref_schema['properties'].items():\n                        if field_def.get('readOnly') is True:\n                            readonly_fields.add(field_name)\n            # Check schema-specific properties\n            if 'properties' in item:\n                for field_name, field_def in item['properties'].items():\n                    if field_def.get('readOnly') is True:\n                        readonly_fields.add(field_name)\n    else:\n        # Direct properties (no inheritance)\n        if 'properties' in schema:\n            for field_name, field_def in schema['properties'].items():\n                if field_def.get('readOnly') is True:\n                    readonly_fields.add(field_name)\n\n    return frozenset(readonly_fields)\n\n\n@functools.cache\ndef get_readonly_watch_fields():\n    \"\"\"\n    Extract readOnly field names from Watch schema in OpenAPI spec.\n\n    Returns readOnly fields from WatchBase (uuid, date_created) + Watch-specific readOnly fields.\n\n    Used by:\n    - model/watch_base.py: Track when writable fields are edited\n    - api/Watch.py: Filter readonly fields from PUT requests\n    \"\"\"\n    return _resolve_readonly_fields('Watch')\n\n\n@functools.cache\ndef get_readonly_tag_fields():\n    \"\"\"\n    Extract readOnly field names from Tag schema in OpenAPI spec.\n\n    Returns readOnly fields from WatchBase (uuid, date_created) + Tag-specific readOnly fields.\n    \"\"\"\n    return _resolve_readonly_fields('Tag')\n"
  },
  {
    "path": "changedetectionio/notification/__init__.py",
    "content": "from changedetectionio.model import USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH\n\ndefault_notification_format = 'htmlcolor'\ndefault_notification_body = '{{watch_url}} had a change.\\n---\\n{{diff}}\\n---\\n'\ndefault_notification_title = 'ChangeDetection.io Notification - {{watch_url}}'\n\n# The values (markdown etc) are from apprise NotifyFormat,\n# But to avoid importing the whole heavy module just use the same strings here.\nvalid_notification_formats = {\n    'text': 'Plain Text',\n    'html': 'HTML',\n    'htmlcolor': 'HTML Color',\n    'markdown': 'Markdown to HTML',\n    # Used only for editing a watch (not for global)\n    USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH: USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH\n}\n"
  },
  {
    "path": "changedetectionio/notification/apprise_plugin/__init__.py",
    "content": ""
  },
  {
    "path": "changedetectionio/notification/apprise_plugin/assets.py",
    "content": "from apprise import AppriseAsset\n\n# Refer to:\n# https://github.com/caronc/apprise/wiki/Development_API#the-apprise-asset-object\n\nAPPRISE_APP_ID = \"changedetection.io\"\nAPPRISE_APP_DESC = \"ChangeDetection.io best and simplest website monitoring and change detection\"\nAPPRISE_APP_URL = \"https://changedetection.io\"\nAPPRISE_AVATAR_URL = \"https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/changedetectionio/static/images/avatar-256x256.png\"\n\napprise_asset = AppriseAsset(\n    app_id=APPRISE_APP_ID,\n    app_desc=APPRISE_APP_DESC,\n    app_url=APPRISE_APP_URL,\n    image_url_logo=APPRISE_AVATAR_URL,\n)\n"
  },
  {
    "path": "changedetectionio/notification/apprise_plugin/custom_handlers.py",
    "content": "\"\"\"\nCustom Apprise HTTP Handlers with format= Parameter Support\n\nIMPORTANT: This module works around a limitation in Apprise's @notify decorator.\n\nTHE PROBLEM:\n-------------\nWhen using Apprise's @notify decorator to create custom notification handlers, the\ndecorator creates a CustomNotifyPlugin that uses parse_url(..., simple=True) to parse\nURLs. This simple parsing mode does NOT extract the format= query parameter from the URL\nand set it as a top-level parameter that NotifyBase.__init__ can use to set notify_format.\n\nAs a result:\n1. URL: post://example.com/webhook?format=html\n2. Apprise parses this and sees format=html in qsd (query string dictionary)\n3. But it does NOT extract it and pass it to NotifyBase.__init__\n4. NotifyBase defaults to notify_format=TEXT\n5. When you call apobj.notify(body=\"<html>...\", body_format=\"html\"):\n   - Apprise sees: input format = html, output format (notify_format) = text\n   - Apprise calls convert_between(\"html\", \"text\", body)\n   - This strips all HTML tags, leaving only plain text\n6. Your custom handler receives stripped plain text instead of HTML\n\nTHE SOLUTION:\n-------------\nInstead of using the @notify decorator directly, we:\n1. Manually register custom plugins using plugins.N_MGR.add()\n2. Create a CustomHTTPHandler class that extends CustomNotifyPlugin\n3. Override __init__ to extract format= from qsd and set it as kwargs['format']\n4. Call NotifyBase.__init__ which properly sets notify_format from kwargs['format']\n5. Set up _default_args like CustomNotifyPlugin does for compatibility\n\nThis ensures that when format=html is in the URL:\n- notify_format is set to HTML\n- Apprise sees: input format = html, output format = html\n- No conversion happens (convert_between returns content unchanged)\n- Your custom handler receives the original HTML intact\n\nTESTING:\n--------\nTo verify this works:\n>>> apobj = apprise.Apprise()\n>>> apobj.add('post://localhost:5005/test?format=html')\n>>> for server in apobj:\n...     print(server.notify_format)  # Should print: html (not text)\n>>> apobj.notify(body='<span>Test</span>', body_format='html')\n# Your handler should receive '<span>Test</span>' not 'Test'\n\"\"\"\n\nimport json\nimport re\nfrom urllib.parse import unquote_plus\n\nimport requests\nfrom apprise import plugins\nfrom apprise.decorators.base import CustomNotifyPlugin\nfrom apprise.utils.parse import parse_url as apprise_parse_url, url_assembly\nfrom apprise.utils.logic import dict_full_update\nfrom loguru import logger\nfrom requests.structures import CaseInsensitiveDict\n\nSUPPORTED_HTTP_METHODS = {\"get\", \"post\", \"put\", \"delete\", \"patch\", \"head\"}\n\n\ndef notify_supported_methods(func):\n    \"\"\"Register custom HTTP method handlers that properly support format= parameter.\"\"\"\n    for method in SUPPORTED_HTTP_METHODS:\n        _register_http_handler(method, func)\n        _register_http_handler(f\"{method}s\", func)\n    return func\n\n\ndef _register_http_handler(schema, send_func):\n    \"\"\"Register a custom HTTP handler that extracts format= from URL query parameters.\"\"\"\n\n    # Parse base URL\n    base_url = f\"{schema}://\"\n    base_args = apprise_parse_url(base_url, default_schema=schema, verify_host=False, simple=True)\n\n    class CustomHTTPHandler(CustomNotifyPlugin):\n        secure_protocol = schema\n        service_name = f\"Custom HTTP - {schema.upper()}\"\n        _base_args = base_args\n\n        def __init__(self, **kwargs):\n            # Extract format from qsd and set it as a top-level kwarg\n            # This allows NotifyBase.__init__ to properly set notify_format\n            if 'qsd' in kwargs and 'format' in kwargs['qsd']:\n                kwargs['format'] = kwargs['qsd']['format']\n\n            # Call NotifyBase.__init__ (skip CustomNotifyPlugin.__init__)\n            super(CustomNotifyPlugin, self).__init__(**kwargs)\n\n            # Set up _default_args like CustomNotifyPlugin does\n            self._default_args = {}\n            kwargs.pop(\"secure\", None)\n            dict_full_update(self._default_args, self._base_args)\n            dict_full_update(self._default_args, kwargs)\n            self._default_args[\"url\"] = url_assembly(**self._default_args)\n\n        __send = staticmethod(send_func)\n\n        def send(self, body, title=\"\", notify_type=\"info\", *args, **kwargs):\n            \"\"\"Call the custom send function.\"\"\"\n            try:\n                result = self.__send(\n                    body, title, notify_type,\n                    *args,\n                    meta=self._default_args,\n                    **kwargs\n                )\n                return True if result is None else bool(result)\n            except Exception as e:\n                self.logger.warning(f\"Exception in custom HTTP handler: {e}\")\n                return False\n\n    # Register the plugin\n    plugins.N_MGR.add(\n        plugin=CustomHTTPHandler,\n        schemas=schema,\n        send_func=send_func,\n        url=base_url,\n    )\n\n\ndef _get_auth(parsed_url: dict) -> str | tuple[str, str]:\n    user: str | None = parsed_url.get(\"user\")\n    password: str | None = parsed_url.get(\"password\")\n\n    if user is not None and password is not None:\n        return (unquote_plus(user), unquote_plus(password))\n\n    if user is not None:\n        return unquote_plus(user)\n\n    return \"\"\n\n\ndef _get_headers(parsed_url: dict, body: str) -> CaseInsensitiveDict:\n    headers = CaseInsensitiveDict(\n        {unquote_plus(k).title(): unquote_plus(v) for k, v in parsed_url[\"qsd+\"].items()}\n    )\n\n    # If Content-Type is not specified, guess if the body is a valid JSON\n    if headers.get(\"Content-Type\") is None:\n        try:\n            json.loads(body)\n            headers[\"Content-Type\"] = \"application/json; charset=utf-8\"\n        except Exception:\n            pass\n\n    return headers\n\n\ndef _get_params(parsed_url: dict) -> CaseInsensitiveDict:\n    # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation\n    # In Apprise, it relies on prefixing each request arg with \"-\", because it uses say &method=update as a flag for apprise\n    # but here we are making straight requests, so we need todo convert this against apprise's logic\n    params = CaseInsensitiveDict(\n        {\n            unquote_plus(k): unquote_plus(v)\n            for k, v in parsed_url[\"qsd\"].items()\n            if k.strip(\"-\") not in parsed_url[\"qsd-\"]\n            and k.strip(\"+\") not in parsed_url[\"qsd+\"]\n        }\n    )\n\n    return params\n\n\n@notify_supported_methods\ndef apprise_http_custom_handler(\n    body: str,\n    title: str,\n    notify_type: str,\n    meta: dict,\n    body_format: str = None,\n    *args,\n    **kwargs,\n) -> bool:\n\n\n    url: str = meta.get(\"url\")\n    schema: str = meta.get(\"schema\")\n    method: str = re.sub(r\"s$\", \"\", schema).upper()\n\n    # Convert /foobar?+some-header=hello to proper header dictionary\n    parsed_url: dict[str, str | dict | None] | None = apprise_parse_url(url)\n    if parsed_url is None:\n        return False\n\n    auth = _get_auth(parsed_url=parsed_url)\n    headers = _get_headers(parsed_url=parsed_url, body=body)\n    params = _get_params(parsed_url=parsed_url)\n\n    url = re.sub(rf\"^{schema}\", \"https\" if schema.endswith(\"s\") else \"http\", parsed_url.get(\"url\"))\n\n    response = requests.request(\n        method=method,\n        url=url,\n        auth=auth,\n        headers=headers,\n        params=params,\n        data=body.encode(\"utf-8\") if isinstance(body, str) else body,\n    )\n\n    response.raise_for_status()\n\n    logger.info(f\"Successfully sent custom notification to {url}\")\n    return True\n"
  },
  {
    "path": "changedetectionio/notification/apprise_plugin/discord.py",
    "content": "\"\"\"\nCustom Discord plugin for changedetection.io\nExtends Apprise's Discord plugin to support custom colored embeds for removed/added content\n\"\"\"\nfrom apprise.plugins.discord import NotifyDiscord\nfrom apprise.decorators import notify\nfrom apprise.common import NotifyFormat\nfrom loguru import logger\n\n# Import placeholders from changedetection's diff module\nfrom ...diff import (\n    REMOVED_PLACEMARKER_OPEN,\n    REMOVED_PLACEMARKER_CLOSED,\n    ADDED_PLACEMARKER_OPEN,\n    ADDED_PLACEMARKER_CLOSED,\n    CHANGED_PLACEMARKER_OPEN,\n    CHANGED_PLACEMARKER_CLOSED,\n    CHANGED_INTO_PLACEMARKER_OPEN,\n    CHANGED_INTO_PLACEMARKER_CLOSED,\n)\n\n# Discord embed sidebar colors for different change types\nDISCORD_COLOR_UNCHANGED = 8421504   # Gray (#808080)\nDISCORD_COLOR_REMOVED = 16711680    # Red (#FF0000)\nDISCORD_COLOR_ADDED = 65280         # Green (#00FF00)\nDISCORD_COLOR_CHANGED = 16753920    # Orange (#FFA500)\nDISCORD_COLOR_CHANGED_INTO = 3447003  # Blue (#5865F2 - Discord blue)\nDISCORD_COLOR_WARNING = 16776960    # Yellow (#FFFF00)\n\n\nclass NotifyDiscordCustom(NotifyDiscord):\n    \"\"\"\n    Custom Discord notification handler that supports multiple colored embeds\n    for showing removed (red) and added (green) content separately.\n    \"\"\"\n\n    def send(self, body, title=\"\", notify_type=None, attach=None, **kwargs):\n        \"\"\"\n        Override send method to create custom embeds with red/green colors\n        for removed/added content when placeholders are present.\n        \"\"\"\n\n        # Check if body contains our diff placeholders\n        has_removed = REMOVED_PLACEMARKER_OPEN in body\n        has_added = ADDED_PLACEMARKER_OPEN in body\n        has_changed = CHANGED_PLACEMARKER_OPEN in body\n        has_changed_into = CHANGED_INTO_PLACEMARKER_OPEN in body\n\n        # If we have diff placeholders and we're in markdown/html format, create custom embeds\n        if (has_removed or has_added or has_changed or has_changed_into) and self.notify_format in (NotifyFormat.MARKDOWN, NotifyFormat.HTML):\n            return self._send_with_colored_embeds(body, title, notify_type, attach, **kwargs)\n\n        # Otherwise, use the parent class's default behavior\n        return super().send(body, title, notify_type, attach, **kwargs)\n\n    def _send_with_colored_embeds(self, body, title, notify_type, attach, **kwargs):\n        \"\"\"\n        Send Discord message with embeds in the original diff order.\n        Preserves the sequence: unchanged -> removed -> added -> unchanged, etc.\n        \"\"\"\n        from datetime import datetime, timezone\n\n        payload = {\n            \"tts\": self.tts,\n            \"wait\": self.tts is False,\n        }\n\n        if self.flags:\n            payload[\"flags\"] = self.flags\n\n        # Acquire image_url\n        image_url = self.image_url(notify_type)\n\n        if self.avatar and (image_url or self.avatar_url):\n            payload[\"avatar_url\"] = self.avatar_url if self.avatar_url else image_url\n\n        if self.user:\n            payload[\"username\"] = self.user\n\n        # Associate our thread_id with our message\n        params = {\"thread_id\": self.thread_id} if self.thread_id else None\n\n        # Build embeds array preserving order\n        embeds = []\n\n        # Add title as plain bold text in message content (not an embed)\n        if title:\n            payload[\"content\"] = f\"**{title}**\"\n\n        # Parse the body into ordered chunks\n        chunks = self._parse_body_into_chunks(body)\n\n        # Discord limits:\n        # - Max 10 embeds per message\n        # - Max 6000 characters total across all embeds\n        # - Max 4096 characters per embed description\n        max_embeds = 10\n        max_total_chars = 6000\n        max_embed_description = 4096\n\n        # All 10 embed slots are available for content\n        max_content_embeds = max_embeds\n\n        # Start character count\n        total_chars = 0\n\n        # Create embeds from chunks in order (no titles, just color coding)\n        for chunk_type, content in chunks:\n            if not content.strip():\n                continue\n\n            # Truncate individual embed description if needed\n            if len(content) > max_embed_description:\n                content = content[:max_embed_description - 3] + \"...\"\n\n            # Check if we're approaching the embed count limit\n            # We need room for the warning embed, so stop at max_content_embeds - 1\n            current_content_embeds = len(embeds)\n            if current_content_embeds >= max_content_embeds - 1:\n                # Add a truncation notice (this will be the 10th embed)\n                embeds.append({\n                    \"description\": \"⚠️ Content truncated (Discord 10 embed limit reached) - Tip: Select 'Plain Text' or 'HTML' format for longer diffs\",\n                    \"color\": DISCORD_COLOR_WARNING,\n                })\n                break\n\n            # Check if adding this embed would exceed total character limit\n            if total_chars + len(content) > max_total_chars:\n                # Add a truncation notice\n                remaining_chars = max_total_chars - total_chars\n                if remaining_chars > 100:\n                    # Add partial content if we have room\n                    truncated_content = content[:remaining_chars - 100] + \"...\"\n                    embeds.append({\n                        \"description\": truncated_content,\n                        \"color\": (DISCORD_COLOR_UNCHANGED if chunk_type == \"unchanged\"\n                                 else DISCORD_COLOR_REMOVED if chunk_type == \"removed\"\n                                 else DISCORD_COLOR_ADDED),\n                    })\n                embeds.append({\n                    \"description\": \"⚠️ Content truncated (Discord 6000 char limit reached)\\nTip: Select 'Plain Text' or 'HTML' format for longer diffs\",\n                    \"color\": DISCORD_COLOR_WARNING,\n                })\n                break\n\n            if chunk_type == \"unchanged\":\n                embeds.append({\n                    \"description\": content,\n                    \"color\": DISCORD_COLOR_UNCHANGED,\n                })\n            elif chunk_type == \"removed\":\n                embeds.append({\n                    \"description\": content,\n                    \"color\": DISCORD_COLOR_REMOVED,\n                })\n            elif chunk_type == \"added\":\n                embeds.append({\n                    \"description\": content,\n                    \"color\": DISCORD_COLOR_ADDED,\n                })\n            elif chunk_type == \"changed\":\n                # Changed (old value) - use orange to distinguish from pure removal\n                embeds.append({\n                    \"description\": content,\n                    \"color\": DISCORD_COLOR_CHANGED,\n                })\n            elif chunk_type == \"changed_into\":\n                # Changed into (new value) - use blue to distinguish from pure addition\n                embeds.append({\n                    \"description\": content,\n                    \"color\": DISCORD_COLOR_CHANGED_INTO,\n                })\n\n            total_chars += len(content)\n\n        if embeds:\n            payload[\"embeds\"] = embeds\n\n        # Send the payload using parent's _send method\n        if not self._send(payload, params=params):\n            return False\n\n        # Handle attachments if present\n        if attach and self.attachment_support:\n            payload.update({\n                \"tts\": False,\n                \"wait\": True,\n            })\n            payload.pop(\"embeds\", None)\n            payload.pop(\"content\", None)\n            payload.pop(\"allow_mentions\", None)\n\n            for attachment in attach:\n                self.logger.info(f\"Posting Discord Attachment {attachment.name}\")\n                if not self._send(payload, params=params, attach=attachment):\n                    return False\n\n        return True\n\n    def _parse_body_into_chunks(self, body):\n        \"\"\"\n        Parse the body into ordered chunks of (type, content) tuples.\n        Types: \"unchanged\", \"removed\", \"added\", \"changed\", \"changed_into\"\n        Preserves the original order of the diff.\n        \"\"\"\n        chunks = []\n        position = 0\n\n        while position < len(body):\n            # Find the next marker\n            next_removed = body.find(REMOVED_PLACEMARKER_OPEN, position)\n            next_added = body.find(ADDED_PLACEMARKER_OPEN, position)\n            next_changed = body.find(CHANGED_PLACEMARKER_OPEN, position)\n            next_changed_into = body.find(CHANGED_INTO_PLACEMARKER_OPEN, position)\n\n            # Determine which marker comes first\n            if next_removed == -1 and next_added == -1 and next_changed == -1 and next_changed_into == -1:\n                # No more markers, rest is unchanged\n                if position < len(body):\n                    chunks.append((\"unchanged\", body[position:]))\n                break\n\n            # Find the earliest marker\n            next_marker_pos = None\n            next_marker_type = None\n\n            # Compare all marker positions to find the earliest\n            markers = []\n            if next_removed != -1:\n                markers.append((next_removed, \"removed\"))\n            if next_added != -1:\n                markers.append((next_added, \"added\"))\n            if next_changed != -1:\n                markers.append((next_changed, \"changed\"))\n            if next_changed_into != -1:\n                markers.append((next_changed_into, \"changed_into\"))\n\n            if markers:\n                next_marker_pos, next_marker_type = min(markers, key=lambda x: x[0])\n\n            # Add unchanged content before the marker\n            if next_marker_pos > position:\n                chunks.append((\"unchanged\", body[position:next_marker_pos]))\n\n            # Find the closing marker\n            if next_marker_type == \"removed\":\n                open_marker = REMOVED_PLACEMARKER_OPEN\n                close_marker = REMOVED_PLACEMARKER_CLOSED\n            elif next_marker_type == \"added\":\n                open_marker = ADDED_PLACEMARKER_OPEN\n                close_marker = ADDED_PLACEMARKER_CLOSED\n            elif next_marker_type == \"changed\":\n                open_marker = CHANGED_PLACEMARKER_OPEN\n                close_marker = CHANGED_PLACEMARKER_CLOSED\n            else:  # changed_into\n                open_marker = CHANGED_INTO_PLACEMARKER_OPEN\n                close_marker = CHANGED_INTO_PLACEMARKER_CLOSED\n\n            close_pos = body.find(close_marker, next_marker_pos)\n\n            if close_pos == -1:\n                # No closing marker, take rest as this type\n                content = body[next_marker_pos + len(open_marker):]\n                chunks.append((next_marker_type, content))\n                break\n            else:\n                # Extract content between markers\n                content = body[next_marker_pos + len(open_marker):close_pos]\n                chunks.append((next_marker_type, content))\n                position = close_pos + len(close_marker)\n\n        return chunks\n\n\n# Register the custom Discord handler with Apprise\n# This will override the built-in discord:// handler\n@notify(on=\"discord\")\ndef discord_custom_wrapper(body, title, notify_type, meta, body_format=None, *args, **kwargs):\n    \"\"\"\n    Wrapper function to make the custom Discord handler work with Apprise's decorator system.\n    Note: This decorator approach may not work for overriding built-in plugins.\n    The class-based approach above is the proper way to extend NotifyDiscord.\n    \"\"\"\n    logger.info(\"Custom Discord handler called\")\n    # This is here for potential future use with decorator-based registration\n    return True\n"
  },
  {
    "path": "changedetectionio/notification/email_helpers.py",
    "content": "def as_monospaced_html_email(content: str, title: str) -> str:\n    \"\"\"\n    Wraps `content` in a minimal, email-safe HTML template\n    that forces monospace rendering across Gmail, Hotmail, Apple Mail, etc.\n\n    Args:\n        content: The body text (plain text or HTML-like).\n        title: The title plaintext\n    Returns:\n        A complete HTML document string suitable for sending as an email body.\n    \"\"\"\n\n    # All line feed types should be removed and then this function should only be fed <br>'s\n    # Then it works with our <pre> styling without double linefeeds\n    content = content.translate(str.maketrans('', '', '\\r\\n'))\n\n    if title:\n        import html\n        title = html.escape(title)\n    else:\n        title = ''\n    # 2. Full email-safe HTML\n    html_email = f\"\"\"<!DOCTYPE html>\n<html lang=\"en\">\n<head>\n  <meta charset=\"UTF-8\">\n  <meta name=\"x-apple-disable-message-reformatting\">\n  <meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\">\n  <!--[if mso]>\n    <style>\n      body, div, pre, td {{ font-family: \"Courier New\", Courier, monospace !important; }}\n    </style>\n  <![endif]-->\n  <title>{title}\n\n\n  
{content}
\n\n\"\"\"\n    return html_email"
  },
  {
    "path": "changedetectionio/notification/handler.py",
    "content": "\nimport time\nimport re\nimport apprise\nfrom apprise import NotifyFormat\nfrom loguru import logger\nfrom urllib.parse import urlparse\nfrom .apprise_plugin.assets import apprise_asset, APPRISE_AVATAR_URL\nfrom .email_helpers import as_monospaced_html_email\nfrom ..diff import HTML_REMOVED_STYLE, REMOVED_PLACEMARKER_OPEN, REMOVED_PLACEMARKER_CLOSED, ADDED_PLACEMARKER_OPEN, HTML_ADDED_STYLE, \\\n    ADDED_PLACEMARKER_CLOSED, CHANGED_INTO_PLACEMARKER_OPEN, CHANGED_INTO_PLACEMARKER_CLOSED, CHANGED_PLACEMARKER_OPEN, \\\n    CHANGED_PLACEMARKER_CLOSED, HTML_CHANGED_STYLE, HTML_CHANGED_INTO_STYLE\nimport re\n\nfrom ..notification_service import NotificationContextData, add_rendered_diff_to_notification_vars\n\nnewline_re = re.compile(r'\\r\\n|\\r|\\n')\n\ndef markup_text_links_to_html(body):\n    \"\"\"\n    Convert plaintext to HTML with clickable links.\n    Uses Jinja2's escape and Markup for XSS safety.\n    \"\"\"\n    from linkify_it import LinkifyIt\n    from markupsafe import Markup, escape\n\n    linkify = LinkifyIt()\n\n    # Match URLs in the ORIGINAL text (before escaping)\n    matches = linkify.match(body)\n\n    if not matches:\n        # No URLs, just escape everything\n        return Markup(escape(body))\n\n    result = []\n    last_index = 0\n\n    # Process each URL match\n    for match in matches:\n        # Add escaped text before the URL\n        if match.index > last_index:\n            text_part = body[last_index:match.index]\n            result.append(escape(text_part))\n\n        # Add the link with escaped URL (both in href and display)\n        url = match.url\n        result.append(Markup(f'{escape(url)}'))\n\n        last_index = match.last_index\n\n    # Add remaining escaped text\n    if last_index < len(body):\n        result.append(escape(body[last_index:]))\n\n    # Join all parts\n    return str(Markup(''.join(str(part) for part in result)))\n\ndef notification_format_align_with_apprise(n_format : str):\n    \"\"\"\n    Correctly align changedetection's formats with apprise's formats\n    Probably these are the same - but good to be sure.\n    These set the expected OUTPUT format type\n    :param n_format:\n    :return:\n    \"\"\"\n\n    if n_format.startswith('html'):\n        # Apprise only knows 'html' not 'htmlcolor' etc, which shouldnt matter here\n        n_format = NotifyFormat.HTML.value\n    elif n_format.startswith('markdown'):\n        # probably the same but just to be safe\n        n_format = NotifyFormat.MARKDOWN.value\n    elif n_format.startswith('text'):\n        # probably the same but just to be safe\n        n_format = NotifyFormat.TEXT.value\n    else:\n        n_format = NotifyFormat.TEXT.value\n\n    return n_format\n\n\ndef apply_html_color_to_body(n_body: str):\n    # https://github.com/dgtlmoon/changedetection.io/issues/821#issuecomment-1241837050\n    n_body = n_body.replace(REMOVED_PLACEMARKER_OPEN,\n                            f'')\n    n_body = n_body.replace(REMOVED_PLACEMARKER_CLOSED, f'')\n    n_body = n_body.replace(ADDED_PLACEMARKER_OPEN,\n                            f'')\n    n_body = n_body.replace(ADDED_PLACEMARKER_CLOSED, f'')\n    # Handle changed/replaced lines (old → new)\n    n_body = n_body.replace(CHANGED_PLACEMARKER_OPEN,\n                            f'')\n    n_body = n_body.replace(CHANGED_PLACEMARKER_CLOSED, f'')\n    n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_OPEN,\n                            f'')\n    n_body = n_body.replace(CHANGED_INTO_PLACEMARKER_CLOSED, f'')\n    return n_body\n\ndef apply_discord_markdown_to_body(n_body):\n    \"\"\"\n    Discord does not support  but it supports non-standard ~~strikethrough~~\n    :param n_body:\n    :return:\n    \"\"\"\n    import re\n    # Define the mapping between your placeholders and markdown markers\n    replacements = [\n        (REMOVED_PLACEMARKER_OPEN, '~~', REMOVED_PLACEMARKER_CLOSED, '~~'),\n        (ADDED_PLACEMARKER_OPEN, '**', ADDED_PLACEMARKER_CLOSED, '**'),\n        (CHANGED_PLACEMARKER_OPEN, '~~', CHANGED_PLACEMARKER_CLOSED, '~~'),\n        (CHANGED_INTO_PLACEMARKER_OPEN, '**', CHANGED_INTO_PLACEMARKER_CLOSED, '**'),\n    ]\n    # So that the markdown gets added without any whitespace following it which would break it\n    for open_tag, open_md, close_tag, close_md in replacements:\n        # Regex: match opening tag, optional whitespace, capture the content, optional whitespace, then closing tag\n        pattern = re.compile(\n            re.escape(open_tag) + r'(\\s*)(.*?)?(\\s*)' + re.escape(close_tag),\n            flags=re.DOTALL\n        )\n        n_body = pattern.sub(lambda m: f\"{m.group(1)}{open_md}{m.group(2)}{close_md}{m.group(3)}\", n_body)\n    return n_body\n\ndef apply_standard_markdown_to_body(n_body):\n    \"\"\"\n    Apprise does not support ~~strikethrough~~ but it will convert  to HTML strikethrough.\n    :param n_body:\n    :return:\n    \"\"\"\n    import re\n    # Define the mapping between your placeholders and markdown markers\n    replacements = [\n        (REMOVED_PLACEMARKER_OPEN, '', REMOVED_PLACEMARKER_CLOSED, ''),\n        (ADDED_PLACEMARKER_OPEN, '**', ADDED_PLACEMARKER_CLOSED, '**'),\n        (CHANGED_PLACEMARKER_OPEN, '', CHANGED_PLACEMARKER_CLOSED, ''),\n        (CHANGED_INTO_PLACEMARKER_OPEN, '**', CHANGED_INTO_PLACEMARKER_CLOSED, '**'),\n    ]\n\n    # So that the markdown gets added without any whitespace following it which would break it\n    for open_tag, open_md, close_tag, close_md in replacements:\n        # Regex: match opening tag, optional whitespace, capture the content, optional whitespace, then closing tag\n        pattern = re.compile(\n            re.escape(open_tag) + r'(\\s*)(.*?)?(\\s*)' + re.escape(close_tag),\n            flags=re.DOTALL\n        )\n        n_body = pattern.sub(lambda m: f\"{m.group(1)}{open_md}{m.group(2)}{close_md}{m.group(3)}\", n_body)\n    return n_body\n\n\ndef replace_placemarkers_in_text(text, url, requested_output_format):\n    \"\"\"\n    Replace diff placemarkers in text based on the URL service type and requested output format.\n    Used for both notification title and body to ensure consistent placeholder replacement.\n\n    :param text: The text to process\n    :param url: The notification URL (to detect service type)\n    :param requested_output_format: The output format (html, htmlcolor, markdown, text, etc.)\n    :return: Processed text with placemarkers replaced\n    \"\"\"\n    if not text:\n        return text\n\n    if url.startswith('tgram://'):\n        # Telegram only supports a limited subset of HTML\n        # Use strikethrough for removed content, bold for added content\n        text = text.replace(REMOVED_PLACEMARKER_OPEN, '')\n        text = text.replace(REMOVED_PLACEMARKER_CLOSED, '')\n        text = text.replace(ADDED_PLACEMARKER_OPEN, '')\n        text = text.replace(ADDED_PLACEMARKER_CLOSED, '')\n        # Handle changed/replaced lines (old → new)\n        text = text.replace(CHANGED_PLACEMARKER_OPEN, '')\n        text = text.replace(CHANGED_PLACEMARKER_CLOSED, '')\n        text = text.replace(CHANGED_INTO_PLACEMARKER_OPEN, '')\n        text = text.replace(CHANGED_INTO_PLACEMARKER_CLOSED, '')\n    elif (url.startswith('discord://') or url.startswith('https://discordapp.com/api/webhooks')\n          or url.startswith('https://discord.com/api')) and requested_output_format == 'html':\n        # Discord doesn't support HTML, use Discord markdown\n        text = apply_discord_markdown_to_body(n_body=text)\n    elif requested_output_format == 'htmlcolor':\n        # https://github.com/dgtlmoon/changedetection.io/issues/821#issuecomment-1241837050\n        text = text.replace(REMOVED_PLACEMARKER_OPEN, f'')\n        text = text.replace(REMOVED_PLACEMARKER_CLOSED, f'')\n        text = text.replace(ADDED_PLACEMARKER_OPEN, f'')\n        text = text.replace(ADDED_PLACEMARKER_CLOSED, f'')\n        # Handle changed/replaced lines (old → new)\n        text = text.replace(CHANGED_PLACEMARKER_OPEN, f'')\n        text = text.replace(CHANGED_PLACEMARKER_CLOSED, f'')\n        text = text.replace(CHANGED_INTO_PLACEMARKER_OPEN, f'')\n        text = text.replace(CHANGED_INTO_PLACEMARKER_CLOSED, f'')\n    elif requested_output_format == 'markdown':\n        # Markdown to HTML - Apprise will convert this to HTML\n        text = apply_standard_markdown_to_body(n_body=text)\n    else:\n        # plaintext, html, and default - use simple text markers\n        text = text.replace(REMOVED_PLACEMARKER_OPEN, '(removed) ')\n        text = text.replace(REMOVED_PLACEMARKER_CLOSED, '')\n        text = text.replace(ADDED_PLACEMARKER_OPEN, '(added) ')\n        text = text.replace(ADDED_PLACEMARKER_CLOSED, '')\n        text = text.replace(CHANGED_PLACEMARKER_OPEN, f'(changed) ')\n        text = text.replace(CHANGED_PLACEMARKER_CLOSED, f'')\n        text = text.replace(CHANGED_INTO_PLACEMARKER_OPEN, f'(into) ')\n        text = text.replace(CHANGED_INTO_PLACEMARKER_CLOSED, f'')\n\n    return text\n\ndef apply_service_tweaks(url, n_body, n_title, requested_output_format):\n\n    logger.debug(f\"Applying markup in '{requested_output_format}' mode\")\n\n    # Re 323 - Limit discord length to their 2000 char limit total or it wont send.\n    # Because different notifications may require different pre-processing, run each sequentially :(\n    # 2000 bytes minus -\n    #     200 bytes for the overhead of the _entire_ json payload, 200 bytes for {tts, wait, content} etc headers\n    #     Length of URL - Incase they specify a longer custom avatar_url\n\n    if not n_body or not n_body.strip():\n        return url, n_body, n_title\n\n    # Normalize URL scheme to lowercase to prevent case-sensitivity issues\n    # e.g., \"Discord://webhook\" -> \"discord://webhook\", \"TGRAM://bot123\" -> \"tgram://bot123\"\n    scheme_separator_pos = url.find('://')\n    if scheme_separator_pos > 0:\n        url = url[:scheme_separator_pos].lower() + url[scheme_separator_pos:]\n\n    # So if no avatar_url is specified, add one so it can be correctly calculated into the total payload\n    parsed = urlparse(url)\n    k = '?' if not parsed.query else '&'\n    if url and not 'avatar_url' in url \\\n            and not url.startswith('mail') \\\n            and not url.startswith('post') \\\n            and not url.startswith('get') \\\n            and not url.startswith('delete') \\\n            and not url.startswith('put'):\n        url += k + f\"avatar_url={APPRISE_AVATAR_URL}\"\n\n    # Replace placemarkers in title first (this was the missing piece causing the bug)\n    # Titles are ALWAYS plain text across all notification services (Discord embeds, Slack attachments,\n    # email Subject headers, etc.), so we always use 'text' format for title placemarker replacement\n    # Looking over apprise library it seems that all plugins only expect plain-text.\n    n_title = replace_placemarkers_in_text(n_title, url, 'text')\n\n    if url.startswith('tgram://'):\n        # Telegram only supports a limit subset of HTML, remove the '
' we place in.\n        # re https://github.com/dgtlmoon/changedetection.io/issues/555\n        # @todo re-use an existing library we have already imported to strip all non-allowed tags\n        n_body = n_body.replace('
', '\\n')\n        n_body = n_body.replace('
', '\\n')\n        n_body = newline_re.sub('\\n', n_body)\n\n        # Replace placemarkers for body\n        n_body = replace_placemarkers_in_text(n_body, url, requested_output_format)\n\n        # real limit is 4096, but minus some for extra metadata\n        payload_max_size = 3600\n        body_limit = max(0, payload_max_size - len(n_title))\n        n_title = n_title[0:payload_max_size]\n        n_body = n_body[0:body_limit]\n\n    elif (url.startswith('discord://') or url.startswith('https://discordapp.com/api/webhooks')\n          or url.startswith('https://discord.com/api'))\\\n            and 'html' in requested_output_format:\n        # Discord doesn't support HTML, replace 
 with newlines\n        n_body = n_body.strip().replace('
', '\\n')\n        n_body = n_body.replace('
', '\\n')\n        n_body = newline_re.sub('\\n', n_body)\n\n        # Don't replace placeholders or truncate here - let the custom Discord plugin handle it\n        # The plugin will use embeds (6000 char limit across all embeds) if placeholders are present,\n        # or plain content (2000 char limit) otherwise\n\n        # Only do placeholder replacement if NOT using htmlcolor (which triggers embeds in custom plugin)\n        if requested_output_format == 'html':\n            # No diff placeholders, use Discord markdown for any other formatting\n            # Use Discord markdown: strikethrough for removed, bold for added\n            n_body = replace_placemarkers_in_text(n_body, url, requested_output_format)\n\n            # Apply 2000 char limit for plain content\n            payload_max_size = 1700\n            body_limit = max(0, payload_max_size - len(n_title))\n            n_title = n_title[0:payload_max_size]\n            n_body = n_body[0:body_limit]\n        # else: our custom Discord plugin will convert any placeholders left over into embeds with color bars\n\n    # Is not discord/tgram and they want htmlcolor\n    elif requested_output_format == 'htmlcolor':\n        n_body = replace_placemarkers_in_text(n_body, url, requested_output_format)\n        n_body = newline_re.sub('
\\n', n_body)\n    elif requested_output_format == 'html':\n        n_body = replace_placemarkers_in_text(n_body, url, requested_output_format)\n        n_body = newline_re.sub('
\\n', n_body)\n    elif requested_output_format == 'markdown':\n        # Markdown to HTML - Apprise will convert this to HTML\n        n_body = replace_placemarkers_in_text(n_body, url, requested_output_format)\n\n    else: #plaintext etc default\n        n_body = replace_placemarkers_in_text(n_body, url, requested_output_format)\n\n    return url, n_body, n_title\n\n\ndef process_notification(n_object: NotificationContextData, datastore):\n    from changedetectionio.jinja2_custom import render as jinja_render\n    from . import USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH, default_notification_format, valid_notification_formats\n    # be sure its registered\n    from .apprise_plugin.custom_handlers import apprise_http_custom_handler\n    # Register custom Discord plugin\n    from .apprise_plugin.discord import NotifyDiscordCustom\n\n    if not isinstance(n_object, NotificationContextData):\n        raise TypeError(f\"Expected NotificationContextData, got {type(n_object)}\")\n\n    now = time.time()\n    if n_object.get('notification_timestamp'):\n        logger.trace(f\"Time since queued {now-n_object['notification_timestamp']:.3f}s\")\n\n    # Insert variables into the notification content\n    notification_parameters = create_notification_parameters(n_object, datastore)\n\n    requested_output_format = n_object.get('notification_format', default_notification_format)\n    logger.debug(f\"Requested notification output format: '{requested_output_format}'\")\n\n    # If we arrived with 'System default' then look it up\n    if requested_output_format == USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH:\n        # Initially text or whatever\n        requested_output_format = datastore.data['settings']['application'].get('notification_format', default_notification_format)\n\n    requested_output_format_original = requested_output_format\n\n    # Now clean it up so it fits perfectly with apprise\n    requested_output_format = notification_format_align_with_apprise(n_format=requested_output_format)\n\n    logger.trace(f\"Complete notification body including Jinja and placeholders calculated in  {time.time() - now:.2f}s\")\n\n    # https://github.com/caronc/apprise/wiki/Development_LogCapture\n    # Anything higher than or equal to WARNING (which covers things like Connection errors)\n    # raise it as an exception\n\n    sent_objs = []\n\n    if 'as_async' in n_object:\n        apprise_asset.async_mode = n_object.get('as_async')\n\n    apobj = apprise.Apprise(debug=True, asset=apprise_asset)\n\n    # Override Apprise's built-in Discord plugin with our custom one\n    # This allows us to use colored embeds for diff content\n    # First remove the built-in discord plugin, then add our custom one\n    apprise.plugins.N_MGR.remove('discord')\n    apprise.plugins.N_MGR.add(NotifyDiscordCustom, schemas='discord')\n\n    if not n_object.get('notification_urls'):\n        return None\n\n    n_object.update(add_rendered_diff_to_notification_vars(\n        notification_scan_text=n_object.get('notification_body', '')+n_object.get('notification_title', ''),\n        current_snapshot=n_object.get('current_snapshot'),\n        prev_snapshot=n_object.get('prev_snapshot'),\n        # Should always be false for 'text' mode or its too hard to read\n        # But otherwise, this could be some setting\n        word_diff=False if requested_output_format_original == 'text' else True,\n        )\n    )\n\n    with (apprise.LogCapture(level=apprise.logging.DEBUG) as logs):\n        for url in n_object['notification_urls']:\n\n            n_body = jinja_render(template_str=n_object.get('notification_body', ''), **notification_parameters)\n            n_title = jinja_render(template_str=n_object.get('notification_title', ''), **notification_parameters)\n\n            if n_object.get('markup_text_links_to_html_links'):\n                n_body = markup_text_links_to_html(body=n_body)\n\n            url = url.strip()\n            if not url or url.startswith('#'):\n                logger.debug(f\"Skipping commented out or empty notification URL - '{url}'\")\n                continue\n\n            logger.info(f\">> Process Notification: AppRise start notifying '{url}'\")\n            url = jinja_render(template_str=url, **notification_parameters)\n\n            # If it's a plaintext document, and they want HTML type email/alerts, so it needs to be escaped\n            watch_mime_type = n_object.get('watch_mime_type')\n            if watch_mime_type and 'text/' in watch_mime_type.lower() and not 'html' in watch_mime_type.lower():\n                if 'html' in requested_output_format:\n                    from markupsafe import escape\n                    n_body = str(escape(n_body))\n\n            if 'html' in requested_output_format:\n                # Since the n_body is always some kind of text from the 'diff' engine, attempt to preserve whitespaces that get sent to the HTML output\n                # But only where its more than 1 consecutive whitespace, otherwise \"and this\" becomes \"and this\" etc which is too much.\n                n_body = n_body.replace('  ', '  ')\n\n            (url, n_body, n_title) = apply_service_tweaks(url=url, n_body=n_body, n_title=n_title, requested_output_format=requested_output_format_original)\n\n            apprise_input_format = \"NO-THANKS-WE-WILL-MANAGE-ALL-OF-THIS\"\n\n            if not 'format=' in url:\n                parsed_url = urlparse(url)\n                prefix_add_to_url = '?' if not parsed_url.query else '&'\n\n                # THIS IS THE TRICK HOW TO DISABLE APPRISE DOING WEIRD AUTO-CONVERSION WITH BREAKING BR TAGS ETC\n                if 'html' in requested_output_format:\n                    url = f\"{url}{prefix_add_to_url}format={NotifyFormat.HTML.value}\"\n                    apprise_input_format = NotifyFormat.HTML.value\n                elif 'text' in requested_output_format:\n                    url = f\"{url}{prefix_add_to_url}format={NotifyFormat.TEXT.value}\"\n                    apprise_input_format = NotifyFormat.TEXT.value\n\n                elif requested_output_format == NotifyFormat.MARKDOWN.value:\n                    # Convert markdown to HTML ourselves since not all plugins do this\n                    from apprise.conversion import markdown_to_html\n                    # Make sure there are paragraph breaks around horizontal rules\n                    n_body = n_body.replace('---', '\\n\\n---\\n\\n')\n                    n_body = markdown_to_html(n_body)\n                    url = f\"{url}{prefix_add_to_url}format={NotifyFormat.HTML.value}\"\n                    requested_output_format = NotifyFormat.HTML.value\n                    apprise_input_format = NotifyFormat.HTML.value  # Changed from MARKDOWN to HTML\n\n            else:\n                # ?format was IN the apprise URL, they are kind of on their own here, we will try our best\n                if 'format=html' in url:\n                    n_body = newline_re.sub('
\\r\\n', n_body)\n                    # This will also prevent apprise from doing conversion\n                    apprise_input_format = NotifyFormat.HTML.value\n                    requested_output_format = NotifyFormat.HTML.value\n                elif 'format=text' in url:\n                    apprise_input_format = NotifyFormat.TEXT.value\n                    requested_output_format = NotifyFormat.TEXT.value\n\n#@todo on null:// (only if its a 1 url with null) probably doesnt need to actually .add/setup/etc\n            sent_objs.append({'title': n_title,\n                              'body': n_body,\n                              'url': url,\n                              # So that we can do a null:// call and get back exactly what would have been sent\n                              'original_context': n_object })\n\n            if not url.startswith('null://'):\n                apobj.add(url)\n\n            # Since the output is always based on the plaintext of the 'diff' engine, wrap it nicely.\n            # It should always be similar to the 'history' part of the UI.\n            if url.startswith('mail') and 'html' in requested_output_format:\n                if not ' Tag settings > Global settings\n    \"\"\"\n    from changedetectionio.notification import (\n        USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH,\n        default_notification_body,\n        default_notification_title\n    )\n\n    # Would be better if this was some kind of Object where Watch can reference the parent datastore etc\n    v = watch.get(var_name)\n    if v and not watch.get('notification_muted'):\n        if var_name == 'notification_format' and v == USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH:\n            return datastore.data['settings']['application'].get('notification_format')\n\n        return v\n\n    tags = datastore.get_all_tags_for_watch(uuid=watch.get('uuid'))\n    if tags:\n        for tag_uuid, tag in tags.items():\n            v = tag.get(var_name)\n            if v and not tag.get('notification_muted'):\n                return v\n\n    if datastore.data['settings']['application'].get(var_name):\n        return datastore.data['settings']['application'].get(var_name)\n\n    # Otherwise could be defaults\n    if var_name == 'notification_format':\n        return USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH\n    if var_name == 'notification_body':\n        return default_notification_body\n    if var_name == 'notification_title':\n        return default_notification_title\n\n    return None\n\n\nclass FormattableTimestamp(str):\n    \"\"\"\n    A str subclass representing a formatted datetime. As a plain string it renders\n    with the default format, but can also be called with a custom format argument\n    in Jinja2 templates:\n\n        {{ change_datetime }}                        → '2024-01-15 10:30:00 UTC'\n        {{ change_datetime(format='%Y') }}           → '2024'\n        {{ change_datetime(format='%A') }}           → 'Monday'\n        {{ change_datetime(format='%Y-%m-%d') }}     → '2024-01-15'\n\n    Being a str subclass means it is natively JSON serializable.\n    \"\"\"\n    _DEFAULT_FORMAT = '%Y-%m-%d %H:%M:%S %Z'\n\n    def __new__(cls, timestamp):\n        dt = datetime.datetime.fromtimestamp(int(timestamp), tz=pytz.UTC)\n        local_tz = datetime.datetime.now().astimezone().tzinfo\n        dt_local = dt.astimezone(local_tz)\n        try:\n            formatted = dt_local.strftime(cls._DEFAULT_FORMAT)\n        except Exception:\n            formatted = dt_local.isoformat()\n        instance = super().__new__(cls, formatted)\n        instance._dt = dt_local\n        return instance\n\n    def __call__(self, format=_DEFAULT_FORMAT):\n        try:\n            return self._dt.strftime(format)\n        except Exception:\n            return self._dt.isoformat()\n\n\nclass FormattableDiff(str):\n    \"\"\"\n    A str subclass representing a rendered diff. As a plain string it renders\n    with the default options for that variant, but can be called with custom\n    arguments in Jinja2 templates:\n\n        {{ diff }}                                    → default diff output\n        {{ diff(lines=5) }}                           → truncate to 5 lines\n        {{ diff(added_only=true) }}                   → only show added lines\n        {{ diff(removed_only=true) }}                 → only show removed lines\n        {{ diff(context=3) }}                         → 3 lines of context around changes\n        {{ diff(word_diff=false) }}                   → line-level diff instead of word-level\n        {{ diff(lines=10, added_only=true) }}         → combine args\n        {{ diff_added(lines=5) }}                     → works on any diff_* variant too\n\n    Being a str subclass means it is natively JSON serializable.\n    \"\"\"\n    def __new__(cls, prev_snapshot, current_snapshot, **base_kwargs):\n        if prev_snapshot or current_snapshot:\n            from changedetectionio import diff as diff_module\n            rendered = diff_module.render_diff(prev_snapshot, current_snapshot, **base_kwargs)\n        else:\n            rendered = ''\n        instance = super().__new__(cls, rendered)\n        instance._prev = prev_snapshot\n        instance._current = current_snapshot\n        instance._base_kwargs = base_kwargs\n        return instance\n\n    def __call__(self, lines=None, added_only=False, removed_only=False, context=0,\n                 word_diff=None, case_insensitive=False, ignore_junk=False):\n        from changedetectionio import diff as diff_module\n        kwargs = dict(self._base_kwargs)\n\n        if added_only:\n            kwargs['include_removed'] = False\n        if removed_only:\n            kwargs['include_added'] = False\n        if context:\n            kwargs['context_lines'] = int(context)\n        if word_diff is not None:\n            kwargs['word_diff'] = bool(word_diff)\n        if case_insensitive:\n            kwargs['case_insensitive'] = True\n        if ignore_junk:\n            kwargs['ignore_junk'] = True\n\n        result = diff_module.render_diff(self._prev or '', self._current or '', **kwargs)\n\n        if lines is not None:\n            result = '\\n'.join(result.splitlines()[:int(lines)])\n\n        return result\n\n\n\n# What is passed around as notification context, also used as the complete list of valid {{ tokens }}\nclass NotificationContextData(dict):\n    def __init__(self, initial_data=None, **kwargs):\n        # ValidateJinja2Template() validates against the keynames of this dict to check for valid tokens in the body (user submission)\n        super().__init__({\n            'base_url': None,\n            'change_datetime': FormattableTimestamp(time.time()),\n            'current_snapshot': None,\n            'diff': FormattableDiff('', ''),\n            'diff_clean': FormattableDiff('', '', include_change_type_prefix=False),\n            'diff_added': FormattableDiff('', '', include_removed=False),\n            'diff_added_clean': FormattableDiff('', '', include_removed=False, include_change_type_prefix=False),\n            'diff_full': FormattableDiff('', '', include_equal=True),\n            'diff_full_clean': FormattableDiff('', '', include_equal=True, include_change_type_prefix=False),\n            'diff_patch': FormattableDiff('', '', patch_format=True),\n            'diff_removed': FormattableDiff('', '', include_added=False),\n            'diff_removed_clean': FormattableDiff('', '', include_added=False, include_change_type_prefix=False),\n            'diff_url': None,\n            'markup_text_links_to_html_links': False, # If automatic conversion of plaintext to HTML should happen\n            'notification_timestamp': time.time(),\n            'prev_snapshot': None,\n            'preview_url': None,\n            'screenshot': None,\n            'timestamp_from': None,\n            'timestamp_to': None,\n            'triggered_text': None,\n            'uuid': 'XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX',  # Converted to 'watch_uuid' in create_notification_parameters\n            'watch_mime_type': None,\n            'watch_tag': None,\n            'watch_title': None,\n            'watch_url': 'https://WATCH-PLACE-HOLDER/',\n            'watch_uuid': 'XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX',  # Converted to 'watch_uuid' in create_notification_parameters\n        })\n\n        # Apply any initial data passed in\n        self.update({'watch_uuid': self.get('uuid')})\n        if initial_data:\n            self.update(initial_data)\n\n        # Apply any keyword arguments\n        if kwargs:\n            self.update(kwargs)\n\n        n_format = self.get('notification_format')\n        if n_format and not valid_notification_formats.get(n_format):\n            raise ValueError(f'Invalid notification format: \"{n_format}\"')\n\n    def set_random_for_validation(self):\n        import random, string\n        \"\"\"Randomly fills all dict keys with random strings (for validation/testing). \n        So we can test the output in the notification body\n        \"\"\"\n        for key in self.keys():\n            if key in ['uuid', 'time', 'watch_uuid', 'change_datetime'] or key.startswith('diff'):\n                continue\n            rand_str = 'RANDOM-PLACEHOLDER-'+''.join(random.choices(string.ascii_letters + string.digits, k=12))\n            self[key] = rand_str\n\n    def __setitem__(self, key, value):\n        if key == 'notification_format' and isinstance(value, str) and not value.startswith('RANDOM-PLACEHOLDER-'):\n            if not valid_notification_formats.get(value):\n                raise ValueError(f'Invalid notification format: \"{value}\"')\n\n        super().__setitem__(key, value)\n\ndef add_rendered_diff_to_notification_vars(notification_scan_text:str, prev_snapshot:str, current_snapshot:str, word_diff:bool):\n    \"\"\"\n    Efficiently renders only the diff placeholders that are actually used in the notification text.\n\n    Scans the notification template for diff placeholder usage (diff, diff_added, diff_clean, etc.)\n    and only renders those specific variants, avoiding expensive render_diff() calls for unused placeholders.\n    Uses LRU caching to avoid duplicate renders when multiple placeholders share the same arguments.\n\n    Args:\n        notification_scan_text: The notification template text to scan for placeholders\n        prev_snapshot: Previous version of content for diff comparison\n        current_snapshot: Current version of content for diff comparison\n        word_diff: Whether to use word-level (True) or line-level (False) diffing\n\n    Returns:\n        dict: Only the diff placeholders that were found in notification_scan_text, with rendered content\n    \"\"\"\n    import re\n\n    now = time.time()\n\n    # Define base kwargs for each diff variant — these become the stored defaults\n    # on the FormattableDiff object, so {{ diff(lines=5) }} overrides on top of them\n    diff_specs = {\n        'diff': {'word_diff': word_diff},\n        'diff_clean': {'word_diff': word_diff, 'include_change_type_prefix': False},\n        'diff_added': {'word_diff': word_diff, 'include_removed': False},\n        'diff_added_clean': {'word_diff': word_diff, 'include_removed': False, 'include_change_type_prefix': False},\n        'diff_full': {'word_diff': word_diff, 'include_equal': True},\n        'diff_full_clean': {'word_diff': word_diff, 'include_equal': True, 'include_change_type_prefix': False},\n        'diff_patch': {'word_diff': word_diff, 'patch_format': True},\n        'diff_removed': {'word_diff': word_diff, 'include_added': False},\n        'diff_removed_clean': {'word_diff': word_diff, 'include_added': False, 'include_change_type_prefix': False},\n    }\n\n    ret = {}\n    rendered_count = 0\n    # Only create FormattableDiff objects for diff keys actually used in the notification text\n    for key in NotificationContextData().keys():\n        if key.startswith('diff') and key in diff_specs:\n            # Check if this placeholder is actually used in the notification text\n            pattern = rf\"(? etc depending on the notification format\n\n    if watch:\n        n_object.update(watch.extra_notification_token_values())\n\n    return n_object\n\nclass NotificationService:\n    \"\"\"\n    Standalone notification service that handles all notification functionality\n    previously embedded in the update_worker class\n    \"\"\"\n    \n    def __init__(self, datastore, notification_q):\n        self.datastore = datastore\n        self.notification_q = notification_q\n    \n    def queue_notification_for_watch(self, n_object: NotificationContextData, watch, date_index_from=-2, date_index_to=-1):\n        \"\"\"\n        Queue a notification for a watch with full diff rendering and template variables\n        \"\"\"\n        from changedetectionio.notification import USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH\n\n        if not isinstance(n_object, NotificationContextData):\n            raise TypeError(f\"Expected NotificationContextData, got {type(n_object)}\")\n\n        dates = []\n        trigger_text = ''\n\n        if watch:\n            watch_history = watch.history\n            dates = list(watch_history.keys())\n            trigger_text = watch.get('trigger_text', [])\n\n        # Add text that was triggered\n        if len(dates):\n            snapshot_contents = watch.get_history_snapshot(timestamp=dates[-1])\n        else:\n            snapshot_contents = \"No snapshot/history available, the watch should fetch atleast once.\"\n\n        # If we ended up here with \"System default\"\n        if n_object.get('notification_format') == USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH:\n            n_object['notification_format'] = self.datastore.data['settings']['application'].get('notification_format')\n\n\n        triggered_text = ''\n        if len(trigger_text):\n            from . import html_tools\n            triggered_text = html_tools.get_triggered_text(content=snapshot_contents, trigger_text=trigger_text)\n            if triggered_text:\n                triggered_text = '\\n'.join(triggered_text)\n\n        # Could be called as a 'test notification' with only 1 snapshot available\n        prev_snapshot = \"Example text: example test\\nExample text: change detection is cool\\nExample text: some more examples\\n\"\n        current_snapshot = \"Example text: example test\\nExample text: change detection is fantastic\\nExample text: even more examples\\nExample text: a lot more examples\"\n\n        if len(dates) > 1:\n            prev_snapshot = watch.get_history_snapshot(timestamp=dates[date_index_from])\n            current_snapshot = watch.get_history_snapshot(timestamp=dates[date_index_to])\n\n\n        n_object.update(set_basic_notification_vars(current_snapshot=current_snapshot,\n                                                    prev_snapshot=prev_snapshot,\n                                                    watch=watch,\n                                                    triggered_text=triggered_text,\n                                                    timestamp_changed=dates[date_index_to]))\n\n        if self.notification_q:\n            logger.debug(\"Queued notification for sending\")\n            self.notification_q.put(n_object)\n        else:\n            logger.debug(\"Not queued, no queue defined. Just returning processed data\")\n            return n_object\n\n    def send_content_changed_notification(self, watch_uuid):\n        \"\"\"\n        Send notification when content changes are detected\n        \"\"\"\n        n_object = NotificationContextData()\n        watch = self.datastore.data['watching'].get(watch_uuid)\n        if not watch:\n            return\n\n        watch_history = watch.history\n        dates = list(watch_history.keys())\n        # Theoretically it's possible that this could be just 1 long,\n        # - In the case that the timestamp key was not unique\n        if len(dates) == 1:\n            raise ValueError(\n                \"History index had 2 or more, but only 1 date loaded, timestamps were not unique? maybe two of the same timestamps got written, needs more delay?\"\n            )\n\n        # Should be a better parent getter in the model object\n\n        # Prefer - Individual watch settings > Tag settings >  Global settings (in that order)\n        # this change probably not needed?\n        n_object['notification_urls'] = _check_cascading_vars(self.datastore, 'notification_urls', watch)\n        n_object['notification_title'] = _check_cascading_vars(self.datastore,'notification_title', watch)\n        n_object['notification_body'] = _check_cascading_vars(self.datastore,'notification_body', watch)\n        n_object['notification_format'] = _check_cascading_vars(self.datastore,'notification_format', watch)\n\n        # (Individual watch) Only prepare to notify if the rules above matched\n        queued = False\n        if n_object and n_object.get('notification_urls'):\n            queued = True\n\n            count = watch.get('notification_alert_count', 0) + 1\n            self.datastore.update_watch(uuid=watch_uuid, update_obj={'notification_alert_count': count})\n\n            self.queue_notification_for_watch(n_object=n_object, watch=watch)\n\n        return queued\n\n    def send_filter_failure_notification(self, watch_uuid):\n        \"\"\"\n        Send notification when CSS/XPath filters fail consecutively\n        \"\"\"\n        threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts')\n        watch = self.datastore.data['watching'].get(watch_uuid)\n        if not watch:\n            return\n\n        filter_list = \", \".join(watch['include_filters'])\n        # @todo - This could be a markdown template on the disk, apprise will convert the markdown to HTML+Plaintext parts in the email, and then 'markup_text_links_to_html_links' is not needed\n        body = f\"\"\"Hello,\n\nYour configured CSS/xPath filters of '{filter_list}' for {{{{watch_url}}}} did not appear on the page after {threshold} attempts.\n\nIt's possible the page changed layout and the filter needs updating ( Try the 'Visual Selector' tab )\n\nEdit link: {{{{base_url}}}}/edit/{{{{watch_uuid}}}}\n\nThanks - Your omniscient changedetection.io installation.\n\"\"\"\n\n        n_object = NotificationContextData({\n            'notification_title': 'Changedetection.io - Alert - CSS/xPath filter was not present in the page',\n            'notification_body': body,\n            'notification_format': _check_cascading_vars(self.datastore, 'notification_format', watch),\n        })\n        n_object['markup_text_links_to_html_links'] = n_object.get('notification_format').startswith('html')\n\n        if len(watch['notification_urls']):\n            n_object['notification_urls'] = watch['notification_urls']\n\n        elif len(self.datastore.data['settings']['application']['notification_urls']):\n            n_object['notification_urls'] = self.datastore.data['settings']['application']['notification_urls']\n\n        # Only prepare to notify if the rules above matched\n        if 'notification_urls' in n_object:\n            n_object.update({\n                'watch_url': watch['url'],\n                'uuid': watch_uuid,\n                'screenshot': None\n            })\n            self.notification_q.put(n_object)\n            logger.debug(f\"Sent filter not found notification for {watch_uuid}\")\n        else:\n            logger.debug(f\"NOT sending filter not found notification for {watch_uuid} - no notification URLs\")\n\n    def send_step_failure_notification(self, watch_uuid, step_n):\n        \"\"\"\n        Send notification when browser steps fail consecutively\n        \"\"\"\n        watch = self.datastore.data['watching'].get(watch_uuid, False)\n        if not watch:\n            return\n        threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts')\n\n        step = step_n + 1\n        # @todo - This could be a markdown template on the disk, apprise will convert the markdown to HTML+Plaintext parts in the email, and then 'markup_text_links_to_html_links' is not needed\n\n        # {{{{ }}}} because this will be Jinja2 {{ }} tokens\n        body = f\"\"\"Hello,\n        \nYour configured browser step at position {step} for the web page watch {{{{watch_url}}}} did not appear on the page after {threshold} attempts, did the page change layout?\n\nThe element may have moved and needs editing, or does it need a delay added?\n\nEdit link: {{{{base_url}}}}/edit/{{{{watch_uuid}}}}\n\nThanks - Your omniscient changedetection.io installation.\n\"\"\"\n\n        n_object = NotificationContextData({\n            'notification_title': f\"Changedetection.io - Alert - Browser step at position {step} could not be run\",\n            'notification_body': body,\n            'notification_format': self._check_cascading_vars('notification_format', watch),\n        })\n        n_object['markup_text_links_to_html_links'] = n_object.get('notification_format').startswith('html')\n\n        if len(watch['notification_urls']):\n            n_object['notification_urls'] = watch['notification_urls']\n\n        elif len(self.datastore.data['settings']['application']['notification_urls']):\n            n_object['notification_urls'] = self.datastore.data['settings']['application']['notification_urls']\n\n        # Only prepare to notify if the rules above matched\n        if 'notification_urls' in n_object:\n            n_object.update({\n                'watch_url': watch['url'],\n                'uuid': watch_uuid\n            })\n            self.notification_q.put(n_object)\n            logger.error(f\"Sent step not found notification for {watch_uuid}\")\n\n\n# Convenience functions for creating notification service instances\ndef create_notification_service(datastore, notification_q):\n    \"\"\"\n    Factory function to create a NotificationService instance\n    \"\"\"\n    return NotificationService(datastore, notification_q)"
  },
  {
    "path": "changedetectionio/pluggy_interface.py",
    "content": "import pluggy\nimport os\nimport importlib\nimport sys\nfrom loguru import logger\n\n# Global plugin namespace for changedetection.io\nPLUGIN_NAMESPACE = \"changedetectionio\"\n\nhookspec = pluggy.HookspecMarker(PLUGIN_NAMESPACE)\nhookimpl = pluggy.HookimplMarker(PLUGIN_NAMESPACE)\n\n\nclass ChangeDetectionSpec:\n    \"\"\"Hook specifications for extending changedetection.io functionality.\"\"\"\n\n    @hookspec\n    def ui_edit_stats_extras(watch):\n        \"\"\"Return HTML content to add to the stats tab in the edit view.\n\n        Args:\n            watch: The watch object being edited\n\n        Returns:\n            str: HTML content to be inserted in the stats tab\n        \"\"\"\n        pass\n\n    @hookspec\n    def register_content_fetcher(self):\n        \"\"\"Return a tuple of (fetcher_name, fetcher_class) for content fetcher plugins.\n\n        The fetcher_name should start with 'html_' and the fetcher_class\n        should inherit from changedetectionio.content_fetchers.base.Fetcher\n\n        Returns:\n            tuple: (str: fetcher_name, class: fetcher_class)\n        \"\"\"\n        pass\n\n    @hookspec\n    def fetcher_status_icon(fetcher_name):\n        \"\"\"Return status icon HTML attributes for a content fetcher.\n\n        Args:\n            fetcher_name: The name of the fetcher (e.g., 'html_webdriver', 'html_js_zyte')\n\n        Returns:\n            str: HTML string containing  tags or other status icon elements\n                 Empty string if no custom status icon is needed\n        \"\"\"\n        pass\n\n    @hookspec\n    def plugin_static_path(self):\n        \"\"\"Return the path to the plugin's static files directory.\n\n        Returns:\n            str: Absolute path to the plugin's static directory, or None if no static files\n        \"\"\"\n        pass\n\n    @hookspec\n    def get_itemprop_availability_override(self, content, fetcher_name, fetcher_instance, url):\n        \"\"\"Provide custom implementation of get_itemprop_availability for a specific fetcher.\n\n        This hook allows plugins to provide their own product availability detection\n        when their fetcher is being used. This is called as a fallback when the built-in\n        method doesn't find good data.\n\n        Args:\n            content: The HTML/text content to parse\n            fetcher_name: The name of the fetcher being used (e.g., 'html_js_zyte')\n            fetcher_instance: The fetcher instance that generated the content\n            url: The URL being watched/checked\n\n        Returns:\n            dict or None: Dictionary with availability data:\n                {\n                    'price': float or None,\n                    'availability': str or None,  # e.g., 'in stock', 'out of stock'\n                    'currency': str or None,      # e.g., 'USD', 'EUR'\n                }\n                Or None if this plugin doesn't handle this fetcher or couldn't extract data\n        \"\"\"\n        pass\n\n    @hookspec\n    def plugin_settings_tab(self):\n        \"\"\"Return settings tab information for this plugin.\n\n        This hook allows plugins to add their own settings tab to the settings page.\n        Settings will be saved to a separate JSON file in the datastore directory.\n\n        Returns:\n            dict or None: Dictionary with settings tab information:\n                {\n                    'plugin_id': str,           # Unique identifier (e.g., 'zyte_fetcher')\n                    'tab_label': str,           # Display name for tab (e.g., 'Zyte Fetcher')\n                    'form_class': Form,         # WTForms Form class for the settings\n                    'template_path': str,       # Optional: path to Jinja2 template (relative to plugin)\n                                                # If not provided, a default form renderer will be used\n                }\n                Or None if this plugin doesn't provide settings\n        \"\"\"\n        pass\n\n    @hookspec\n    def register_processor(self):\n        \"\"\"Register an external processor plugin.\n\n        External packages can implement this hook to register custom processors\n        that will be discovered alongside built-in processors.\n\n        Returns:\n            dict or None: Dictionary with processor information:\n                {\n                    'processor_name': str,      # Machine name (e.g., 'osint_recon')\n                    'processor_module': module, # Module containing processor.py\n                    'processor_class': class,   # The perform_site_check class\n                    'metadata': {               # Optional metadata\n                        'name': str,            # Display name\n                        'description': str,     # Description\n                        'processor_weight': int,# Sort weight (lower = higher priority)\n                        'list_badge_text': str, # Badge text for UI\n                    }\n                }\n                Return None if this plugin doesn't provide a processor\n        \"\"\"\n        pass\n\n    @hookspec\n    def update_handler_alter(update_handler, watch, datastore):\n        \"\"\"Modify or wrap the update_handler before it processes a watch.\n\n        This hook is called after the update_handler (perform_site_check instance) is created\n        but before it calls call_browser() and run_changedetection(). Plugins can use this to:\n        - Wrap the handler to add logging/metrics\n        - Modify handler configuration\n        - Add custom preprocessing logic\n\n        Args:\n            update_handler: The perform_site_check instance that will process the watch\n            watch: The watch dict being processed\n            datastore: The application datastore\n\n        Returns:\n            object or None: Return a modified/wrapped handler, or None to keep the original.\n                           If multiple plugins return handlers, they are chained in registration order.\n        \"\"\"\n        pass\n\n    @hookspec\n    def update_finalize(update_handler, watch, datastore, processing_exception):\n        \"\"\"Called after watch processing completes (success or failure).\n\n        This hook is called in the finally block after all processing is complete,\n        allowing plugins to perform cleanup, update metrics, or log final status.\n\n        The plugin can access update_handler.last_logging_insert_id if it was stored\n        during update_handler_alter, and use processing_exception to determine if\n        the processing succeeded or failed.\n\n        Args:\n            update_handler: The perform_site_check instance (may be None if creation failed)\n            watch: The watch dict that was processed (may be None if not loaded)\n            datastore: The application datastore\n            processing_exception: The exception from the main processing block, or None if successful.\n                                 This does NOT include cleanup exceptions - only exceptions from\n                                 the actual watch processing (fetch, diff, etc).\n\n        Returns:\n            None: This hook doesn't return a value\n        \"\"\"\n        pass\n\n\n# Set up Plugin Manager\nplugin_manager = pluggy.PluginManager(PLUGIN_NAMESPACE)\n\n# Register hookspecs\nplugin_manager.add_hookspecs(ChangeDetectionSpec)\n\n# Load plugins from subdirectories\ndef load_plugins_from_directories():\n    # Dictionary of directories to scan for plugins\n    plugin_dirs = {\n        'conditions': os.path.join(os.path.dirname(__file__), 'conditions', 'plugins'),\n        # Add more plugin directories here as needed\n    }\n    \n    # Note: Removed the direct import of example_word_count_plugin as it's now in the conditions/plugins directory\n    \n    for dir_name, dir_path in plugin_dirs.items():\n        if not os.path.exists(dir_path):\n            continue\n            \n        # Get all Python files (excluding __init__.py)\n        for filename in os.listdir(dir_path):\n            if filename.endswith(\".py\") and filename != \"__init__.py\":\n                module_name = filename[:-3]  # Remove .py extension\n                module_path = f\"changedetectionio.{dir_name}.plugins.{module_name}\"\n                \n                try:\n                    module = importlib.import_module(module_path)\n                    # Register the plugin with pluggy\n                    plugin_manager.register(module, module_name)\n                except (ImportError, AttributeError) as e:\n                    print(f\"Error loading plugin {module_name}: {e}\")\n\n# Load plugins\nload_plugins_from_directories()\n\n# Discover installed plugins from external packages (if any)\nplugin_manager.load_setuptools_entrypoints(PLUGIN_NAMESPACE)\n\n# Function to inject datastore into plugins that need it\ndef inject_datastore_into_plugins(datastore):\n    \"\"\"Inject the global datastore into plugins that need access to settings.\n\n    This should be called after plugins are loaded and datastore is initialized.\n\n    Args:\n        datastore: The global ChangeDetectionStore instance\n    \"\"\"\n    for plugin_name, plugin_obj in plugin_manager.list_name_plugin():\n        # Check if plugin has datastore attribute and it's not set\n        if hasattr(plugin_obj, 'datastore'):\n            if plugin_obj.datastore is None:\n                plugin_obj.datastore = datastore\n                logger.debug(f\"Injected datastore into plugin: {plugin_name}\")\n\n# Function to register built-in fetchers - called later from content_fetchers/__init__.py\ndef register_builtin_fetchers():\n    \"\"\"Register built-in content fetchers as internal plugins\n\n    This is called from content_fetchers/__init__.py after all fetchers are imported\n    to avoid circular import issues.\n    \"\"\"\n    from changedetectionio.content_fetchers import requests, playwright, puppeteer, webdriver_selenium\n\n    # Register each built-in fetcher plugin\n    if hasattr(requests, 'requests_plugin'):\n        plugin_manager.register(requests.requests_plugin, 'builtin_requests')\n\n    if hasattr(playwright, 'playwright_plugin'):\n        plugin_manager.register(playwright.playwright_plugin, 'builtin_playwright')\n\n    if hasattr(puppeteer, 'puppeteer_plugin'):\n        plugin_manager.register(puppeteer.puppeteer_plugin, 'builtin_puppeteer')\n\n    if hasattr(webdriver_selenium, 'webdriver_selenium_plugin'):\n        plugin_manager.register(webdriver_selenium.webdriver_selenium_plugin, 'builtin_webdriver_selenium')\n\n# Helper function to collect UI stats extras from all plugins\ndef collect_ui_edit_stats_extras(watch):\n    \"\"\"Collect and combine HTML content from all plugins that implement ui_edit_stats_extras\"\"\"\n    extras_content = []\n\n    # Get all plugins that implement the ui_edit_stats_extras hook\n    results = plugin_manager.hook.ui_edit_stats_extras(watch=watch)\n\n    # If we have results, add them to our content\n    if results:\n        for result in results:\n            if result:  # Skip empty results\n                extras_content.append(result)\n\n    return \"\\n\".join(extras_content) if extras_content else \"\"\n\ndef collect_fetcher_status_icons(fetcher_name):\n    \"\"\"Collect status icon data from all plugins\n\n    Args:\n        fetcher_name: The name of the fetcher (e.g., 'html_webdriver', 'html_js_zyte')\n\n    Returns:\n        dict or None: Icon data dictionary from first matching plugin, or None\n    \"\"\"\n    # Get status icon data from plugins\n    results = plugin_manager.hook.fetcher_status_icon(fetcher_name=fetcher_name)\n\n    # Return first non-None result\n    if results:\n        for result in results:\n            if result and isinstance(result, dict):\n                return result\n\n    return None\n\ndef get_itemprop_availability_from_plugin(content, fetcher_name, fetcher_instance, url):\n    \"\"\"Get itemprop availability data from plugins as a fallback.\n\n    This is called when the built-in get_itemprop_availability doesn't find good data.\n\n    Args:\n        content: The HTML/text content to parse\n        fetcher_name: The name of the fetcher being used (e.g., 'html_js_zyte')\n        fetcher_instance: The fetcher instance that generated the content\n        url: The URL being watched (watch.link - includes Jinja2 evaluation)\n\n    Returns:\n        dict or None: Availability data dictionary from first matching plugin, or None\n    \"\"\"\n    # Get availability data from plugins\n    results = plugin_manager.hook.get_itemprop_availability_override(\n        content=content,\n        fetcher_name=fetcher_name,\n        fetcher_instance=fetcher_instance,\n        url=url\n    )\n\n    # Return first non-None result with actual data\n    if results:\n        for result in results:\n            if result and isinstance(result, dict):\n                # Check if the result has any meaningful data\n                if result.get('price') is not None or result.get('availability'):\n                    return result\n\n    return None\n\n\ndef get_active_plugins():\n    \"\"\"Get a list of active plugins with their descriptions.\n\n    Returns:\n        list: List of dictionaries with plugin information:\n            [\n                {'name': 'plugin_name', 'description': 'Plugin description'},\n                ...\n            ]\n    \"\"\"\n    active_plugins = []\n\n    # Get all registered plugins\n    for plugin_name, plugin_obj in plugin_manager.list_name_plugin():\n        # Skip built-in plugins (they start with 'builtin_')\n        if plugin_name.startswith('builtin_'):\n            continue\n\n        # Get plugin description if available\n        description = None\n        if hasattr(plugin_obj, '__doc__') and plugin_obj.__doc__:\n            description = plugin_obj.__doc__.strip().split('\\n')[0]  # First line only\n        elif hasattr(plugin_obj, 'description'):\n            description = plugin_obj.description\n\n        # Try to get a friendly name from the plugin\n        friendly_name = plugin_name\n        if hasattr(plugin_obj, 'name'):\n            friendly_name = plugin_obj.name\n\n        active_plugins.append({\n            'name': friendly_name,\n            'description': description or 'No description available'\n        })\n\n    return active_plugins\n\n\ndef get_fetcher_capabilities(watch, datastore):\n    \"\"\"Get capability flags for a watch's fetcher.\n\n    Args:\n        watch: The watch object/dict\n        datastore: The datastore to resolve 'system' fetcher\n\n    Returns:\n        dict: Dictionary with capability flags:\n            {\n                'supports_browser_steps': bool,\n                'supports_screenshots': bool,\n                'supports_xpath_element_data': bool\n            }\n    \"\"\"\n    # Get the fetcher name from watch\n    fetcher_name = watch.get('fetch_backend', 'system')\n\n    # Resolve 'system' to actual fetcher\n    if fetcher_name == 'system':\n        fetcher_name = datastore.data['settings']['application'].get('fetch_backend', 'html_requests')\n\n    # Get the fetcher class\n    from changedetectionio import content_fetchers\n\n    # Try to get from built-in fetchers first\n    if hasattr(content_fetchers, fetcher_name):\n        fetcher_class = getattr(content_fetchers, fetcher_name)\n        return {\n            'supports_browser_steps': getattr(fetcher_class, 'supports_browser_steps', False),\n            'supports_screenshots': getattr(fetcher_class, 'supports_screenshots', False),\n            'supports_xpath_element_data': getattr(fetcher_class, 'supports_xpath_element_data', False)\n        }\n\n    # Try to get from plugin-provided fetchers\n    # Query all plugins for registered fetchers\n    plugin_fetchers = plugin_manager.hook.register_content_fetcher()\n    for fetcher_registration in plugin_fetchers:\n        if fetcher_registration:\n            name, fetcher_class = fetcher_registration\n            if name == fetcher_name:\n                return {\n                    'supports_browser_steps': getattr(fetcher_class, 'supports_browser_steps', False),\n                    'supports_screenshots': getattr(fetcher_class, 'supports_screenshots', False),\n                    'supports_xpath_element_data': getattr(fetcher_class, 'supports_xpath_element_data', False)\n                }\n\n    # Default: no capabilities\n    return {\n        'supports_browser_steps': False,\n        'supports_screenshots': False,\n        'supports_xpath_element_data': False\n    }\n\n\ndef get_plugin_settings_tabs():\n    \"\"\"Get all plugin settings tabs.\n\n    Returns:\n        list: List of dictionaries with plugin settings tab information:\n            [\n                {\n                    'plugin_id': str,\n                    'tab_label': str,\n                    'form_class': Form,\n                    'description': str\n                },\n                ...\n            ]\n    \"\"\"\n    tabs = []\n    results = plugin_manager.hook.plugin_settings_tab()\n\n    for result in results:\n        if result and isinstance(result, dict):\n            # Validate required fields\n            if 'plugin_id' in result and 'tab_label' in result and 'form_class' in result:\n                tabs.append(result)\n            else:\n                logger.warning(f\"Invalid plugin settings tab spec: {result}\")\n\n    return tabs\n\n\ndef load_plugin_settings(datastore_path, plugin_id):\n    \"\"\"Load settings for a specific plugin from JSON file.\n\n    Args:\n        datastore_path: Path to the datastore directory\n        plugin_id: Unique identifier for the plugin (e.g., 'zyte_fetcher')\n\n    Returns:\n        dict: Plugin settings, or empty dict if file doesn't exist\n    \"\"\"\n    import json\n    settings_file = os.path.join(datastore_path, f\"{plugin_id}.json\")\n\n    if not os.path.exists(settings_file):\n        return {}\n\n    try:\n        with open(settings_file, 'r', encoding='utf-8') as f:\n            return json.load(f)\n    except Exception as e:\n        logger.error(f\"Failed to load settings for plugin '{plugin_id}': {e}\")\n        return {}\n\n\ndef save_plugin_settings(datastore_path, plugin_id, settings):\n    \"\"\"Save settings for a specific plugin to JSON file.\n\n    Args:\n        datastore_path: Path to the datastore directory\n        plugin_id: Unique identifier for the plugin (e.g., 'zyte_fetcher')\n        settings: Dictionary of settings to save\n\n    Returns:\n        bool: True if save was successful, False otherwise\n    \"\"\"\n    import json\n    settings_file = os.path.join(datastore_path, f\"{plugin_id}.json\")\n\n    try:\n        with open(settings_file, 'w', encoding='utf-8') as f:\n            json.dump(settings, f, indent=2, ensure_ascii=False)\n        logger.info(f\"Saved settings for plugin '{plugin_id}' to {settings_file}\")\n        return True\n    except Exception as e:\n        logger.error(f\"Failed to save settings for plugin '{plugin_id}': {e}\")\n        return False\n\n\ndef get_plugin_template_paths():\n    \"\"\"Get list of plugin template directories for Jinja2 loader.\n\n    Scans both external pluggy plugins and built-in processor plugins.\n\n    Returns:\n        list: List of absolute paths to plugin template directories\n    \"\"\"\n    template_paths = []\n\n    # Add the base processors/templates directory (as absolute path)\n    processors_templates_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'processors', 'templates')\n    if os.path.isdir(processors_templates_dir):\n        template_paths.append(processors_templates_dir)\n        logger.debug(f\"Added base processors template path: {processors_templates_dir}\")\n\n    # Scan built-in processor plugins\n    from changedetectionio.processors import find_processors\n    processor_list = find_processors()\n    for processor_module, processor_name in processor_list:\n        # Each processor is a module, check if it has a templates directory\n        if hasattr(processor_module, '__file__'):\n            processor_file = processor_module.__file__\n            if processor_file:\n                # Get the processor directory (e.g., processors/image_ssim_diff/)\n                processor_dir = os.path.dirname(os.path.abspath(processor_file))\n                templates_dir = os.path.join(processor_dir, 'templates')\n                if os.path.isdir(templates_dir):\n                    template_paths.append(templates_dir)\n                    logger.debug(f\"Added processor template path: {templates_dir}\")\n\n    # Get all registered external pluggy plugins\n    for plugin_name, plugin_obj in plugin_manager.list_name_plugin():\n        # Check if plugin has a templates directory\n        if hasattr(plugin_obj, '__file__'):\n            plugin_file = plugin_obj.__file__\n        elif hasattr(plugin_obj, '__module__'):\n            # Get the module file\n            module = sys.modules.get(plugin_obj.__module__)\n            if module and hasattr(module, '__file__'):\n                plugin_file = module.__file__\n            else:\n                continue\n        else:\n            continue\n\n        if plugin_file:\n            plugin_dir = os.path.dirname(os.path.abspath(plugin_file))\n            templates_dir = os.path.join(plugin_dir, 'templates')\n            if os.path.isdir(templates_dir):\n                template_paths.append(templates_dir)\n                logger.debug(f\"Added plugin template path: {templates_dir}\")\n\n    return template_paths\n\n\ndef apply_update_handler_alter(update_handler, watch, datastore):\n    \"\"\"Apply update_handler_alter hooks from all plugins.\n\n    Allows plugins to wrap or modify the update_handler before it processes a watch.\n    Multiple plugins can chain modifications - each plugin receives the result from\n    the previous plugin.\n\n    Args:\n        update_handler: The perform_site_check instance to potentially modify\n        watch: The watch dict being processed\n        datastore: The application datastore\n\n    Returns:\n        object: The (potentially modified/wrapped) update_handler\n    \"\"\"\n    # Get all plugins that implement the update_handler_alter hook\n    results = plugin_manager.hook.update_handler_alter(\n        update_handler=update_handler,\n        watch=watch,\n        datastore=datastore\n    )\n\n    # Chain results - each plugin gets the result from the previous one\n    current_handler = update_handler\n    if results:\n        for result in results:\n            if result is not None:\n                logger.debug(f\"Plugin modified update_handler for watch {watch.get('uuid')}\")\n                current_handler = result\n\n    return current_handler\n\n\ndef apply_update_finalize(update_handler, watch, datastore, processing_exception):\n    \"\"\"Apply update_finalize hooks from all plugins.\n\n    Called in the finally block after watch processing completes, allowing plugins\n    to perform cleanup, update metrics, or log final status.\n\n    Args:\n        update_handler: The perform_site_check instance (may be None)\n        watch: The watch dict that was processed (may be None)\n        datastore: The application datastore\n        processing_exception: The exception from processing, or None if successful\n\n    Returns:\n        None\n    \"\"\"\n    try:\n        # Call all plugins that implement the update_finalize hook\n        plugin_manager.hook.update_finalize(\n            update_handler=update_handler,\n            watch=watch,\n            datastore=datastore,\n            processing_exception=processing_exception\n        )\n    except Exception as e:\n        # Don't let plugin errors crash the worker\n        logger.error(f\"Error in update_finalize hook: {e}\")\n        logger.exception(f\"update_finalize hook exception details:\")"
  },
  {
    "path": "changedetectionio/processors/README.md",
    "content": "# Change detection post-processors\n\nThe concept here is to be able to switch between different domain specific problems to solve.\n\n- `text_json_diff` The traditional text and JSON comparison handler\n- `restock_diff` Only cares about detecting if a product looks like it has some text that suggests that it's out of stock, otherwise assumes that it's in stock.\n\nSome suggestions for the future\n\n- `graphical` \n\n## API schema extension (`api.yaml`)\n\nA processor can extend the Watch/Tag API schema by placing an `api.yaml` alongside its `__init__.py`.\nDefine a `components.schemas.processor_config_` entry and it will be merged into `WatchBase` at startup,\nmaking `processor_config_` a valid field on all watch create/update API calls.\nThe fully merged spec is served live at `/api/v1/full-spec`.\n\nSee `restock_diff/api.yaml` for a working example.\n\n## Todo\n\n- Make each processor return a extra list of sub-processed (so you could configure a single processor in different ways)\n- move restock_diff to its own pip/github repo\n"
  },
  {
    "path": "changedetectionio/processors/__init__.py",
    "content": "from functools import lru_cache\nfrom loguru import logger\nfrom flask_babel import gettext, get_locale\nimport importlib\nimport inspect\nimport os\nimport pkgutil\n\ndef find_sub_packages(package_name):\n    \"\"\"\n    Find all sub-packages within the given package.\n\n    :param package_name: The name of the base package to scan for sub-packages.\n    :return: A list of sub-package names.\n    \"\"\"\n    package = importlib.import_module(package_name)\n    return [name for _, name, is_pkg in pkgutil.iter_modules(package.__path__) if is_pkg]\n\n\n@lru_cache(maxsize=1)\ndef find_processors():\n    \"\"\"\n    Find all subclasses of DifferenceDetectionProcessor in the specified package.\n    Results are cached to avoid repeated discovery.\n\n    :param package_name: The name of the package to scan for processor modules.\n    :return: A list of (module, class) tuples.\n    \"\"\"\n    package_name = \"changedetectionio.processors\"  # Name of the current package/module\n\n    processors = []\n    sub_packages = find_sub_packages(package_name)\n    from changedetectionio.processors.base import difference_detection_processor\n\n    for sub_package in sub_packages:\n        module_name = f\"{package_name}.{sub_package}.processor\"\n        try:\n            module = importlib.import_module(module_name)\n\n            # Iterate through all classes in the module\n            for name, obj in inspect.getmembers(module, inspect.isclass):\n                # Only register classes that are actually defined in this module (not imported)\n                if (issubclass(obj, difference_detection_processor) and\n                    obj is not difference_detection_processor and\n                    obj.__module__ == module.__name__):\n                    processors.append((module, sub_package))\n                    break  # Only need one processor per module\n        except (ModuleNotFoundError, ImportError) as e:\n            logger.warning(f\"Failed to import module {module_name}: {e} (find_processors())\")\n\n    # Discover plugin processors via pluggy\n    try:\n        from changedetectionio.pluggy_interface import plugin_manager\n        plugin_results = plugin_manager.hook.register_processor()\n\n        for result in plugin_results:\n            if result and isinstance(result, dict):\n                processor_module = result.get('processor_module')\n                processor_name = result.get('processor_name')\n\n                if processor_module and processor_name:\n                    processors.append((processor_module, processor_name))\n                    plugin_path = getattr(processor_module, '__file__', 'unknown location')\n                    logger.info(f\"Registered plugin processor: {processor_name} from {plugin_path}\")\n    except Exception as e:\n        logger.warning(f\"Error loading plugin processors: {e}\")\n\n    return processors\n\n\ndef get_parent_module(module):\n    module_name = module.__name__\n    if '.' not in module_name:\n        return None  # Top-level module has no parent\n    parent_module_name = module_name.rsplit('.', 1)[0]\n    try:\n        return importlib.import_module(parent_module_name)\n    except Exception as e:\n        pass\n\n    return False\n\n\n\ndef get_custom_watch_obj_for_processor(processor_name):\n    from changedetectionio.model import Watch\n    watch_class = Watch.model\n    processor_classes = find_processors()\n    custom_watch_obj = next((tpl for tpl in processor_classes if tpl[1] == processor_name), None)\n    if custom_watch_obj:\n        # Parent of .processor.py COULD have its own Watch implementation\n        parent_module = get_parent_module(custom_watch_obj[0])\n        if hasattr(parent_module, 'Watch'):\n            watch_class = parent_module.Watch\n\n    return watch_class\n\n\ndef find_processor_module(processor_name):\n    \"\"\"\n    Find the processor module by name.\n\n    Args:\n        processor_name: Processor machine name (e.g., 'image_ssim_diff')\n\n    Returns:\n        module: The processor's parent module, or None if not found\n    \"\"\"\n    processor_classes = find_processors()\n    processor_tuple = next((tpl for tpl in processor_classes if tpl[1] == processor_name), None)\n\n    if processor_tuple:\n        # Return the parent module (the package containing processor.py)\n        return get_parent_module(processor_tuple[0])\n\n    return None\n\n\ndef get_processor_module(processor_name):\n    \"\"\"\n    Get the actual processor module (with perform_site_check class) by name.\n    Works for both built-in and plugin processors.\n\n    Args:\n        processor_name: Processor machine name (e.g., 'text_json_diff', 'osint_recon')\n\n    Returns:\n        module: The processor module containing perform_site_check, or None if not found\n    \"\"\"\n    processor_classes = find_processors()\n    processor_tuple = next((tpl for tpl in processor_classes if tpl[1] == processor_name), None)\n\n    if processor_tuple:\n        # Return the actual processor module (first element of tuple)\n        return processor_tuple[0]\n\n    return None\n\n\ndef get_processor_submodule(processor_name, submodule_name):\n    \"\"\"\n    Get an optional submodule from a processor (e.g., 'difference', 'extract', 'preview').\n    Works for both built-in and plugin processors.\n\n    Args:\n        processor_name: Processor machine name (e.g., 'text_json_diff', 'osint_recon')\n        submodule_name: Name of the submodule (e.g., 'difference', 'extract', 'preview')\n\n    Returns:\n        module: The submodule if it exists, or None if not found\n    \"\"\"\n    processor_classes = find_processors()\n    processor_tuple = next((tpl for tpl in processor_classes if tpl[1] == processor_name), None)\n\n    if not processor_tuple:\n        return None\n\n    processor_module = processor_tuple[0]\n    parent_module = get_parent_module(processor_module)\n\n    if not parent_module:\n        return None\n\n    # Try to import the submodule\n    try:\n        # For built-in processors: changedetectionio.processors.text_json_diff.difference\n        # For plugin processors: changedetectionio_osint.difference\n        parent_module_name = parent_module.__name__\n        submodule_full_name = f\"{parent_module_name}.{submodule_name}\"\n        return importlib.import_module(submodule_full_name)\n    except (ModuleNotFoundError, ImportError):\n        return None\n\n\n@lru_cache(maxsize=1)\ndef get_plugin_processor_metadata():\n    \"\"\"Get metadata from plugin processors.\"\"\"\n    metadata = {}\n    try:\n        from changedetectionio.pluggy_interface import plugin_manager\n        plugin_results = plugin_manager.hook.register_processor()\n\n        for result in plugin_results:\n            if result and isinstance(result, dict):\n                processor_name = result.get('processor_name')\n                meta = result.get('metadata', {})\n                if processor_name:\n                    metadata[processor_name] = meta\n    except Exception as e:\n        logger.warning(f\"Error getting plugin processor metadata: {e}\")\n    return metadata\n\n@lru_cache(maxsize=32)\ndef _available_processors_cached(locale_str):\n    \"\"\"\n    Internal cached function that includes locale in cache key.\n    This ensures translations are cached per-language instead of globally.\n\n    :param locale_str: The locale string (e.g., 'en', 'it', 'zh')\n    :return: A list of tuples (processor_name, translated_description, weight)\n    \"\"\"\n    processor_classes = find_processors()\n\n    # Check if DISABLED_PROCESSORS env var is set\n    disabled_processors_env = os.getenv('DISABLED_PROCESSORS', 'image_ssim_diff').strip()\n    disabled_processors = []\n    if disabled_processors_env:\n        # Parse comma-separated list and strip whitespace\n        disabled_processors = [p.strip() for p in disabled_processors_env.split(',') if p.strip()]\n        logger.info(f\"DISABLED_PROCESSORS set, disabling: {disabled_processors}\")\n\n    available = []\n    plugin_metadata = get_plugin_processor_metadata()\n\n    for module, sub_package_name in processor_classes:\n        # Skip disabled processors\n        if sub_package_name in disabled_processors:\n            logger.debug(f\"Skipping processor '{sub_package_name}' (in DISABLED_PROCESSORS)\")\n            continue\n\n        # Check if this is a plugin processor\n        if sub_package_name in plugin_metadata:\n            meta = plugin_metadata[sub_package_name]\n            description = gettext(meta.get('name', sub_package_name))\n            # Plugin processors start from weight 10 to separate them from built-in processors\n            weight = 100 + meta.get('processor_weight', 0)\n        else:\n            # Try to get the 'name' attribute from the processor module first\n            if hasattr(module, 'name'):\n                description = gettext(module.name)\n            else:\n                # Fall back to processor_description from parent module's __init__.py\n                parent_module = get_parent_module(module)\n                if parent_module and hasattr(parent_module, 'processor_description'):\n                    description = gettext(parent_module.processor_description)\n                else:\n                    # Final fallback to a readable name\n                    description = sub_package_name.replace('_', ' ').title()\n\n            # Get weight for sorting (lower weight = higher in list)\n            weight = 0  # Default weight for processors without explicit weight\n\n            # Check processor module itself first\n            if hasattr(module, 'processor_weight'):\n                weight = module.processor_weight\n            else:\n                # Fall back to parent module (package __init__.py)\n                parent_module = get_parent_module(module)\n                if parent_module and hasattr(parent_module, 'processor_weight'):\n                    weight = parent_module.processor_weight\n\n        available.append((sub_package_name, description, weight))\n\n    # Sort by weight (lower weight = appears first)\n    available.sort(key=lambda x: x[2])\n\n    # Return as tuples without weight (for backwards compatibility)\n    return [(name, desc) for name, desc, weight in available]\n\ndef available_processors():\n    \"\"\"\n    Get a list of processors by name and description for the UI elements.\n    Can be filtered via DISABLED_PROCESSORS environment variable (comma-separated list).\n\n    This function delegates to a locale-aware cached version to ensure translations\n    are cached per-language instead of globally.\n\n    :return: A list of tuples (processor_name, translated_description)\n    \"\"\"\n    # Get current locale and use it as cache key\n    # Convert Babel Locale object to string for use as cache key\n    locale = get_locale()\n    locale_str = str(locale) if locale else 'en'\n    return _available_processors_cached(locale_str)\n\n\ndef get_default_processor():\n    \"\"\"\n    Get the default processor to use when none is specified.\n    Returns the first available processor based on weight (lowest weight = highest priority).\n    This ensures forms auto-select a valid processor even when DISABLED_PROCESSORS filters the list.\n\n    :return: The processor name string (e.g., 'text_json_diff')\n    \"\"\"\n    available = available_processors()\n    if available:\n        return available[0][0]  # Return the processor name from first tuple\n    return 'text_json_diff'  # Fallback if somehow no processors are available\n\n\ndef get_processor_badge_texts():\n    \"\"\"\n    Get a dictionary mapping processor names to their list_badge_text values.\n    Translations are applied based on the current request locale.\n\n    :return: A dict mapping processor name to badge text (e.g., {'text_json_diff': 'Text', 'restock_diff': 'Restock'})\n    \"\"\"\n    processor_classes = find_processors()\n    badge_texts = {}\n\n    for module, sub_package_name in processor_classes:\n        # Try to get the 'list_badge_text' attribute from the processor module\n        if hasattr(module, 'list_badge_text'):\n            badge_texts[sub_package_name] = gettext(module.list_badge_text)\n        else:\n            # Fall back to parent module's __init__.py\n            parent_module = get_parent_module(module)\n            if parent_module and hasattr(parent_module, 'list_badge_text'):\n                badge_texts[sub_package_name] = gettext(parent_module.list_badge_text)\n\n    return badge_texts\n\n\ndef get_processor_descriptions():\n    \"\"\"\n    Get a dictionary mapping processor names to their description/name values.\n    Translations are applied based on the current request locale.\n\n    :return: A dict mapping processor name to description (e.g., {'text_json_diff': 'Webpage Text/HTML, JSON and PDF changes'})\n    \"\"\"\n    processor_classes = find_processors()\n    descriptions = {}\n\n    for module, sub_package_name in processor_classes:\n        # Try to get the 'name' or 'description' attribute from the processor module first\n        if hasattr(module, 'name'):\n            descriptions[sub_package_name] = gettext(module.name)\n        elif hasattr(module, 'description'):\n            descriptions[sub_package_name] = gettext(module.description)\n        else:\n            # Fall back to parent module's __init__.py\n            parent_module = get_parent_module(module)\n            if parent_module and hasattr(parent_module, 'processor_description'):\n                descriptions[sub_package_name] = gettext(parent_module.processor_description)\n            elif parent_module and hasattr(parent_module, 'name'):\n                descriptions[sub_package_name] = gettext(parent_module.name)\n            else:\n                # Final fallback to a readable name\n                descriptions[sub_package_name] = sub_package_name.replace('_', ' ').title()\n\n    return descriptions\n\n\ndef generate_processor_badge_colors(processor_name):\n    \"\"\"\n    Generate consistent colors for a processor badge based on its name.\n    Uses a hash of the processor name to generate pleasing, accessible colors\n    for both light and dark modes.\n\n    :param processor_name: The processor name (e.g., 'text_json_diff')\n    :return: A dict with 'light' and 'dark' color schemes, each containing 'bg' and 'color'\n    \"\"\"\n    import hashlib\n\n    # Generate a consistent hash from the processor name\n    hash_obj = hashlib.md5(processor_name.encode('utf-8'))\n    hash_int = int(hash_obj.hexdigest()[:8], 16)\n\n    # Generate hue from hash (0-360)\n    hue = hash_int % 360\n\n    # Light mode: pastel background with darker text\n    light_saturation = 60 + (hash_int % 25)  # 60-85%\n    light_lightness = 85 + (hash_int % 10)   # 85-95% - very light\n    text_lightness = 25 + (hash_int % 15)    # 25-40% - dark\n\n    # Dark mode: solid, vibrant colors with white text\n    dark_saturation = 55 + (hash_int % 20)   # 55-75%\n    dark_lightness = 45 + (hash_int % 15)    # 45-60%\n\n    return {\n        'light': {\n            'bg': f'hsl({hue}, {light_saturation}%, {light_lightness}%)',\n            'color': f'hsl({hue}, 50%, {text_lightness}%)'\n        },\n        'dark': {\n            'bg': f'hsl({hue}, {dark_saturation}%, {dark_lightness}%)',\n            'color': '#fff'\n        }\n    }\n\n\n@lru_cache(maxsize=1)\ndef get_processor_badge_css():\n    \"\"\"\n    Generate CSS for all processor badges with auto-generated colors.\n    This creates CSS rules for both light and dark modes for each processor.\n\n    :return: A string containing CSS rules for all processor badges\n    \"\"\"\n    processor_classes = find_processors()\n    css_rules = []\n\n    for module, sub_package_name in processor_classes:\n        colors = generate_processor_badge_colors(sub_package_name)\n\n        # Light mode rule\n        css_rules.append(\n            f\".processor-badge-{sub_package_name} {{\\n\"\n            f\"  background-color: {colors['light']['bg']};\\n\"\n            f\"  color: {colors['light']['color']};\\n\"\n            f\"}}\"\n        )\n\n        # Dark mode rule\n        css_rules.append(\n            f\"html[data-darkmode=\\\"true\\\"] .processor-badge-{sub_package_name} {{\\n\"\n            f\"  background-color: {colors['dark']['bg']};\\n\"\n            f\"  color: {colors['dark']['color']};\\n\"\n            f\"}}\"\n        )\n\n    return '\\n\\n'.join(css_rules)\n\n\ndef save_processor_config(datastore, watch_uuid, config_data):\n    \"\"\"\n    Save processor-specific configuration to JSON file.\n\n    This is a shared helper function used by both the UI edit form and API endpoints\n    to consistently handle processor configuration storage.\n\n    Args:\n        datastore: The application datastore instance\n        watch_uuid: UUID of the watch\n        config_data: Dictionary of configuration data to save (with processor_config_* prefix removed)\n\n    Returns:\n        bool: True if saved successfully, False otherwise\n    \"\"\"\n    if not config_data:\n        return True\n\n    try:\n        from changedetectionio.processors.base import difference_detection_processor\n\n        # Get processor name from watch\n        watch = datastore.data['watching'].get(watch_uuid)\n        if not watch:\n            logger.error(f\"Cannot save processor config: watch {watch_uuid} not found\")\n            return False\n\n        processor_name = watch.get('processor', 'text_json_diff')\n\n        # Create a processor instance to access config methods\n        processor_instance = difference_detection_processor(datastore, watch_uuid)\n\n        # Use processor name as filename so each processor keeps its own config\n        config_filename = f'{processor_name}.json'\n        processor_instance.update_extra_watch_config(config_filename, config_data)\n\n        logger.debug(f\"Saved processor config to {config_filename}: {config_data}\")\n        return True\n\n    except Exception as e:\n        logger.error(f\"Failed to save processor config: {e}\")\n        return False\n\n\ndef extract_processor_config_from_form_data(form_data):\n    \"\"\"\n    Extract processor_config_* fields from form data and return separate dicts.\n\n    This is a shared helper function used by both the UI edit form and API endpoints\n    to consistently handle processor configuration extraction.\n\n    IMPORTANT: This function modifies form_data in-place by removing processor_config_* fields.\n\n    Args:\n        form_data: Dictionary of form data (will be modified in-place)\n\n    Returns:\n        dict: Dictionary of processor config data (with processor_config_* prefix removed)\n    \"\"\"\n    processor_config_data = {}\n\n    # Use list() to create a copy of keys since we're modifying the dict\n    for field_name in list(form_data.keys()):\n        if field_name.startswith('processor_config_'):\n            config_key = field_name.replace('processor_config_', '')\n            # Save all values (including empty strings) to allow explicit clearing of settings\n            processor_config_data[config_key] = form_data[field_name]\n            # Remove from form_data to prevent it from reaching datastore\n            del form_data[field_name]\n\n    return processor_config_data\n\n"
  },
  {
    "path": "changedetectionio/processors/base.py",
    "content": "import asyncio\nimport re\nimport hashlib\n\nfrom changedetectionio.browser_steps.browser_steps import browser_steps_get_valid_steps\nfrom changedetectionio.content_fetchers.base import Fetcher\nfrom changedetectionio.strtobool import strtobool\nfrom changedetectionio.validate_url import is_private_hostname\nfrom copy import deepcopy\nfrom abc import abstractmethod\nimport os\nfrom urllib.parse import urlparse\nfrom loguru import logger\n\nSCREENSHOT_FORMAT_JPEG = 'JPEG'\nSCREENSHOT_FORMAT_PNG = 'PNG'\n\nclass difference_detection_processor():\n    browser_steps = None\n    datastore = None\n    fetcher = None\n    screenshot = None\n    watch = None\n    xpath_data = None\n    preferred_proxy = None\n    screenshot_format = SCREENSHOT_FORMAT_JPEG\n    last_raw_content_checksum = None\n\n    def __init__(self, datastore, watch_uuid):\n        self.datastore = datastore\n        self.watch_uuid = watch_uuid\n\n        # Create a stable snapshot of the watch for processing\n        # Why deepcopy?\n        # 1. Prevents \"dict changed during iteration\" errors if watch is modified during processing\n        # 2. Preserves Watch object with properties (.link, .is_pdf, etc.) - can't use dict()\n        # 3. Safe now: Watch.__deepcopy__() shares datastore ref (no memory leak) but copies dict data\n        self.watch = deepcopy(self.datastore.data['watching'].get(watch_uuid))\n\n        # Generic fetcher that should be extended (requests, playwright etc)\n        self.fetcher = Fetcher()\n\n        # Load the last raw content checksum from file\n        self.read_last_raw_content_checksum()\n\n    def update_last_raw_content_checksum(self, checksum):\n        \"\"\"\n        Save the raw content MD5 checksum to file.\n        This is used for skip logic - avoid reprocessing if raw HTML unchanged.\n        \"\"\"\n        if not checksum:\n            return\n\n        watch = self.datastore.data['watching'].get(self.watch_uuid)\n        if not watch:\n            return\n\n        data_dir = watch.data_dir\n        if not data_dir:\n            return\n\n        watch.ensure_data_dir_exists()\n        checksum_file = os.path.join(data_dir, 'last-checksum.txt')\n\n        try:\n            with open(checksum_file, 'w', encoding='utf-8') as f:\n                f.write(checksum)\n            self.last_raw_content_checksum = checksum\n        except IOError as e:\n            logger.warning(f\"Failed to write checksum file for {self.watch_uuid}: {e}\")\n\n    def read_last_raw_content_checksum(self):\n        \"\"\"\n        Read the last raw content MD5 checksum from file.\n        Returns None if file doesn't exist (first run) or can't be read.\n        \"\"\"\n        watch = self.datastore.data['watching'].get(self.watch_uuid)\n        if not watch:\n            self.last_raw_content_checksum = None\n            return\n\n        data_dir = watch.data_dir\n        if not data_dir:\n            self.last_raw_content_checksum = None\n            return\n\n        checksum_file = os.path.join(data_dir, 'last-checksum.txt')\n\n        if not os.path.isfile(checksum_file):\n            self.last_raw_content_checksum = None\n            return\n\n        try:\n            with open(checksum_file, 'r', encoding='utf-8') as f:\n                self.last_raw_content_checksum = f.read().strip()\n        except IOError as e:\n            logger.warning(f\"Failed to read checksum file for {self.watch_uuid}: {e}\")\n            self.last_raw_content_checksum = None\n\n\n    async def validate_iana_url(self):\n        \"\"\"Pre-flight SSRF check — runs DNS lookup in executor to avoid blocking the event loop.\n        Covers all fetchers (requests, playwright, puppeteer, plugins) since every fetch goes\n        through call_browser().\n        \"\"\"\n        if strtobool(os.getenv('ALLOW_IANA_RESTRICTED_ADDRESSES', 'false')):\n            return\n        parsed = urlparse(self.watch.link)\n        if not parsed.hostname:\n            return\n        loop = asyncio.get_running_loop()\n        if await loop.run_in_executor(None, is_private_hostname, parsed.hostname):\n            raise Exception(\n                f\"Fetch blocked: '{self.watch.link}' resolves to a private/reserved IP address. \"\n                f\"Set ALLOW_IANA_RESTRICTED_ADDRESSES=true to allow.\"\n            )\n\n    async def call_browser(self, preferred_proxy_id=None):\n\n        from requests.structures import CaseInsensitiveDict\n\n        url = self.watch.link\n\n        # Protect against file:, file:/, file:// access, check the real \"link\" without any meta \"source:\" etc prepended.\n        if re.search(r'^file:', url.strip(), re.IGNORECASE):\n            if not strtobool(os.getenv('ALLOW_FILE_URI', 'false')):\n                raise Exception(\n                    \"file:// type access is denied for security reasons.\"\n                )\n\n        await self.validate_iana_url()\n\n        # Requests, playwright, other browser via wss:// etc, fetch_extra_something\n        prefer_fetch_backend = self.watch.get('fetch_backend', 'system')\n\n        # Proxy ID \"key\"\n        preferred_proxy_id = preferred_proxy_id if preferred_proxy_id else self.datastore.get_preferred_proxy_for_watch(\n            uuid=self.watch.get('uuid'))\n\n        # Pluggable content self.fetcher\n        if not prefer_fetch_backend or prefer_fetch_backend == 'system':\n            prefer_fetch_backend = self.datastore.data['settings']['application'].get('fetch_backend')\n\n        # In the case that the preferred fetcher was a browser config with custom connection URL..\n        # @todo - on save watch, if its extra_browser_ then it should be obvious it will use playwright (like if its requests now..)\n        custom_browser_connection_url = None\n        if prefer_fetch_backend.startswith('extra_browser_'):\n            (t, key) = prefer_fetch_backend.split('extra_browser_')\n            connection = list(\n                filter(lambda s: (s['browser_name'] == key), self.datastore.data['settings']['requests'].get('extra_browsers', [])))\n            if connection:\n                prefer_fetch_backend = 'html_webdriver'\n                custom_browser_connection_url = connection[0].get('browser_connection_url')\n\n        # PDF should be html_requests because playwright will serve it up (so far) in a embedded page\n        # @todo https://github.com/dgtlmoon/changedetection.io/issues/2019\n        # @todo needs test to or a fix\n        if self.watch.is_pdf:\n            prefer_fetch_backend = \"html_requests\"\n\n        # Grab the right kind of 'fetcher', (playwright, requests, etc)\n        from changedetectionio import content_fetchers\n        if hasattr(content_fetchers, prefer_fetch_backend):\n            # @todo TEMPORARY HACK - SWITCH BACK TO PLAYWRIGHT FOR BROWSERSTEPS\n            if prefer_fetch_backend == 'html_webdriver' and self.watch.has_browser_steps:\n                # This is never supported in selenium anyway\n                logger.warning(\n                    \"Using playwright fetcher override for possible puppeteer request in browsersteps, because puppetteer:browser steps is incomplete.\")\n                from changedetectionio.content_fetchers.playwright import fetcher as playwright_fetcher\n                fetcher_obj = playwright_fetcher\n            else:\n                fetcher_obj = getattr(content_fetchers, prefer_fetch_backend)\n        else:\n            # What it referenced doesnt exist, Just use a default\n            fetcher_obj = getattr(content_fetchers, \"html_requests\")\n\n        proxy_url = None\n        if preferred_proxy_id:\n            # Custom browser endpoints should NOT have a proxy added\n            if not prefer_fetch_backend.startswith('extra_browser_'):\n                proxy_url = self.datastore.proxy_list.get(preferred_proxy_id).get('url')\n                logger.debug(f\"Selected proxy key '{preferred_proxy_id}' as proxy URL '{proxy_url}' for {url}\")\n            else:\n                logger.debug(\"Skipping adding proxy data when custom Browser endpoint is specified. \")\n\n        logger.debug(f\"Using proxy '{proxy_url}' for {self.watch['uuid']}\")\n\n        # Now call the fetcher (playwright/requests/etc) with arguments that only a fetcher would need.\n        # When browser_connection_url is None, it method should default to working out whats the best defaults (os env vars etc)\n        self.fetcher = fetcher_obj(proxy_override=proxy_url,\n                                   custom_browser_connection_url=custom_browser_connection_url,\n                                   screenshot_format=self.screenshot_format\n                                   )\n\n        if self.watch.has_browser_steps:\n            self.fetcher.browser_steps = browser_steps_get_valid_steps(self.watch.get('browser_steps', []))\n            self.fetcher.browser_steps_screenshot_path = os.path.join(self.datastore.datastore_path, self.watch.get('uuid'))\n\n        # Tweak the base config with the per-watch ones\n        from changedetectionio.jinja2_custom import render as jinja_render\n        request_headers = CaseInsensitiveDict()\n\n        ua = self.datastore.data['settings']['requests'].get('default_ua')\n        if ua and ua.get(prefer_fetch_backend):\n            request_headers.update({'User-Agent': ua.get(prefer_fetch_backend)})\n\n        request_headers.update(self.watch.get('headers', {}))\n        request_headers.update(self.datastore.get_all_base_headers())\n        request_headers.update(self.datastore.get_all_headers_in_textfile_for_watch(uuid=self.watch.get('uuid')))\n\n        # https://github.com/psf/requests/issues/4525\n        # Requests doesnt yet support brotli encoding, so don't put 'br' here, be totally sure that the user cannot\n        # do this by accident.\n        if 'Accept-Encoding' in request_headers and \"br\" in request_headers['Accept-Encoding']:\n            request_headers['Accept-Encoding'] = request_headers['Accept-Encoding'].replace(', br', '')\n\n        for header_name in request_headers:\n            request_headers.update({header_name: jinja_render(template_str=request_headers.get(header_name))})\n\n        timeout = self.datastore.data['settings']['requests'].get('timeout')\n\n        request_body = self.watch.get('body')\n        if request_body:\n            request_body = jinja_render(template_str=self.watch.get('body'))\n\n        request_method = self.watch.get('method')\n        ignore_status_codes = self.watch.get('ignore_status_codes', False)\n\n        # Configurable per-watch or global extra delay before extracting text (for webDriver types)\n        system_webdriver_delay = self.datastore.data['settings']['application'].get('webdriver_delay', None)\n        if self.watch.get('webdriver_delay'):\n            self.fetcher.render_extract_delay = self.watch.get('webdriver_delay')\n        elif system_webdriver_delay is not None:\n            self.fetcher.render_extract_delay = system_webdriver_delay\n\n        if self.watch.get('webdriver_js_execute_code') is not None and self.watch.get('webdriver_js_execute_code').strip():\n            self.fetcher.webdriver_js_execute_code = self.watch.get('webdriver_js_execute_code')\n\n        # Requests for PDF's, images etc should be passwd the is_binary flag\n        is_binary = self.watch.is_pdf\n\n        # And here we go! call the right browser with browser-specific settings\n        empty_pages_are_a_change = self.datastore.data['settings']['application'].get('empty_pages_are_a_change', False)\n        # All fetchers are now async\n        await self.fetcher.run(\n            current_include_filters=self.watch.get('include_filters'),\n            empty_pages_are_a_change=empty_pages_are_a_change,\n            fetch_favicon=self.watch.favicon_is_expired(),\n            ignore_status_codes=ignore_status_codes,\n            is_binary=is_binary,\n            request_body=request_body,\n            request_headers=request_headers,\n            request_method=request_method,\n            screenshot_format=self.screenshot_format,\n            timeout=timeout,\n            url=url,\n            watch_uuid=self.watch_uuid,\n        )\n\n        # @todo .quit here could go on close object, so we can run JS if change-detected\n        await self.fetcher.quit(watch=self.watch)\n\n        # Sanitize lone surrogates - these can appear when servers return malformed/mixed-encoding\n        # content that gets decoded into surrogate characters (e.g. \\udcad). Without this,\n        # encode('utf-8') raises UnicodeEncodeError downstream in checksums, diffs, file writes, etc.\n        # Covers all fetchers (requests, playwright, puppeteer, selenium) in one place.\n        # Also note: By this point we SHOULD know the original encoding so it can safely convert to utf-8 for the rest of the app.\n        # See: https://github.com/dgtlmoon/changedetection.io/issues/3952\n\n        if self.fetcher.content and isinstance(self.fetcher.content, str):\n            self.fetcher.content = self.fetcher.content.encode('utf-8', errors='replace').decode('utf-8')\n\n        # After init, call run_changedetection() which will do the actual change-detection\n\n    def get_extra_watch_config(self, filename):\n        \"\"\"\n        Read processor-specific JSON config file from watch data directory.\n\n        Args:\n            filename: Name of JSON file (e.g., \"visual_ssim_score.json\")\n\n        Returns:\n            dict: Parsed JSON data, or empty dict if file doesn't exist\n        \"\"\"\n        import json\n        import os\n\n        watch = self.datastore.data['watching'].get(self.watch_uuid)\n        data_dir = watch.data_dir\n\n        if not data_dir:\n            return {}\n\n        filepath = os.path.join(data_dir, filename)\n\n        if not os.path.isfile(filepath):\n            return {}\n\n        try:\n            with open(filepath, 'r', encoding='utf-8') as f:\n                return json.load(f)\n        except (json.JSONDecodeError, IOError) as e:\n            logger.warning(f\"Failed to read extra watch config {filename}: {e}\")\n            return {}\n\n    def update_extra_watch_config(self, filename, data, merge=True):\n        \"\"\"\n        Write processor-specific JSON config file to watch data directory.\n\n        Args:\n            filename: Name of JSON file (e.g., \"visual_ssim_score.json\")\n            data: Dictionary to serialize as JSON\n            merge: If True, merge with existing data; if False, overwrite completely\n        \"\"\"\n        import json\n        import os\n\n        watch = self.datastore.data['watching'].get(self.watch_uuid)\n        data_dir = watch.data_dir\n\n        if not data_dir:\n            logger.warning(f\"Cannot save extra watch config {filename}: no data_dir\")\n            return\n\n        # Ensure directory exists\n        watch.ensure_data_dir_exists()\n\n        filepath = os.path.join(data_dir, filename)\n\n        try:\n            # If merge is enabled, read existing data first\n            existing_data = {}\n            if merge and os.path.isfile(filepath):\n                try:\n                    with open(filepath, 'r', encoding='utf-8') as f:\n                        existing_data = json.load(f)\n                except (json.JSONDecodeError, IOError) as e:\n                    logger.warning(f\"Failed to read existing config for merge: {e}\")\n\n            # Merge new data with existing\n            if merge:\n                existing_data.update(data)\n                data_to_save = existing_data\n            else:\n                data_to_save = data\n\n            # Write the data\n            with open(filepath, 'w', encoding='utf-8') as f:\n                json.dump(data_to_save, f, indent=2)\n        except IOError as e:\n            logger.error(f\"Failed to write extra watch config {filename}: {e}\")\n\n    def get_raw_document_checksum(self):\n        checksum = None\n\n        if self.fetcher.content:\n            checksum = hashlib.md5(self.fetcher.content.encode('utf-8')).hexdigest()\n\n        return checksum\n\n    @abstractmethod\n    def run_changedetection(self, watch, force_reprocess=False):\n        update_obj = {'last_notification_error': False, 'last_error': False}\n        some_data = 'xxxxx'\n        update_obj[\"previous_md5\"] = hashlib.md5(some_data.encode('utf-8')).hexdigest()\n        changed_detected = False\n        return changed_detected, update_obj, ''.encode('utf-8')\n"
  },
  {
    "path": "changedetectionio/processors/exceptions.py",
    "content": "class ProcessorException(Exception):\n    def __init__(self, message=None, status_code=None, url=None, screenshot=None, has_filters=False, html_content='', xpath_data=None):\n        self.message = message\n        self.status_code = status_code\n        self.url = url\n        self.screenshot = screenshot\n        self.has_filters = has_filters\n        self.html_content = html_content\n        self.xpath_data = xpath_data\n        return\n"
  },
  {
    "path": "changedetectionio/processors/extract.py",
    "content": "\"\"\"\nBase data extraction module for all processors.\n\nThis module handles extracting data from watch history using regex patterns\nand exporting to CSV format. This is the default extractor that all processors\n(text_json_diff, restock_diff, etc.) can use by default or override.\n\"\"\"\n\nimport os\nfrom flask_babel import gettext\nfrom loguru import logger\n\n\ndef render_form(watch, datastore, request, url_for, render_template, flash, redirect, extract_form=None):\n    \"\"\"\n    Render the data extraction form.\n\n    Args:\n        watch: The watch object\n        datastore: The ChangeDetectionStore instance\n        request: Flask request object\n        url_for: Flask url_for function\n        render_template: Flask render_template function\n        flash: Flask flash function\n        redirect: Flask redirect function\n        extract_form: Optional pre-built extract form (for error cases)\n\n    Returns:\n        Rendered HTML response with the extraction form\n    \"\"\"\n    from changedetectionio import forms\n\n    uuid = watch.get('uuid')\n\n    # Use provided form or create a new one\n    if extract_form is None:\n        extract_form = forms.extractDataForm(\n            formdata=request.form,\n            data={'extract_regex': request.form.get('extract_regex', '')}\n        )\n\n    # Get error information for the template\n    screenshot_url = watch.get_screenshot()\n\n    is_html_webdriver = watch.fetcher_supports_screenshots\n\n    password_enabled_and_share_is_off = False\n    if datastore.data['settings']['application'].get('password') or os.getenv(\"SALTED_PASS\", False):\n        password_enabled_and_share_is_off = not datastore.data['settings']['application'].get('shared_diff_access')\n\n    # Use the shared default template from processors/templates/\n    # Processors can override this by creating their own extract.py with custom template logic\n    output = render_template(\n        \"extract.html\",\n        uuid=uuid,\n        extract_form=extract_form,\n        watch_a=watch,\n        last_error=watch['last_error'],\n        last_error_screenshot=watch.get_error_snapshot(),\n        last_error_text=watch.get_error_text(),\n        screenshot=screenshot_url,\n        is_html_webdriver=is_html_webdriver,\n        password_enabled_and_share_is_off=password_enabled_and_share_is_off,\n        extra_title=f\" - {watch.label} - Extract Data\",\n        extra_stylesheets=[url_for('static_content', group='styles', filename='diff.css')],\n        pure_menu_fixed=False\n    )\n\n    return output\n\n\ndef process_extraction(watch, datastore, request, url_for, make_response, send_from_directory, flash, redirect, extract_form=None):\n    \"\"\"\n    Process the data extraction request and return CSV file.\n\n    Args:\n        watch: The watch object\n        datastore: The ChangeDetectionStore instance\n        request: Flask request object\n        url_for: Flask url_for function\n        make_response: Flask make_response function\n        send_from_directory: Flask send_from_directory function\n        flash: Flask flash function\n        redirect: Flask redirect function\n        extract_form: Optional pre-built extract form\n\n    Returns:\n        CSV file download response or redirect to form on error\n    \"\"\"\n    from changedetectionio import forms\n\n    uuid = watch.get('uuid')\n\n    # Use provided form or create a new one\n    if extract_form is None:\n        extract_form = forms.extractDataForm(\n            formdata=request.form,\n            data={'extract_regex': request.form.get('extract_regex', '')}\n        )\n\n    if not extract_form.validate():\n        flash(gettext(\"An error occurred, please see below.\"), \"error\")\n        # render_template needs to be imported from Flask for this to work\n        from flask import render_template as flask_render_template\n        return render_form(\n            watch=watch,\n            datastore=datastore,\n            request=request,\n            url_for=url_for,\n            render_template=flask_render_template,\n            flash=flash,\n            redirect=redirect,\n            extract_form=extract_form\n        )\n\n    extract_regex = request.form.get('extract_regex', '').strip()\n    output = watch.extract_regex_from_all_history(extract_regex)\n\n    if output:\n        watch_dir = os.path.join(datastore.datastore_path, uuid)\n        response = make_response(send_from_directory(directory=watch_dir, path=output, as_attachment=True))\n        response.headers['Content-type'] = 'text/csv'\n        response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'\n        response.headers['Pragma'] = 'no-cache'\n        response.headers['Expires'] = \"0\"\n        return response\n\n    flash(gettext('No matches found while scanning all of the watch history for that RegEx.'), 'error')\n    return redirect(url_for('ui.ui_diff.diff_history_page_extract_GET', uuid=uuid))\n"
  },
  {
    "path": "changedetectionio/processors/image_ssim_diff/README.md",
    "content": "# Fast Screenshot Comparison Processor\n\nVisual/screenshot change detection using ultra-fast image comparison algorithms.\n\n## Overview\n\nThis processor uses **OpenCV** by default for screenshot comparison, providing **50-100x faster** performance compared to the previous SSIM implementation while still detecting meaningful visual changes.\n\n## Current Features\n\n- **Ultra-fast OpenCV comparison**: cv2.absdiff with Gaussian blur for noise reduction\n- **MD5 pre-check**: Fast identical image detection before expensive comparison\n- **Configurable sensitivity**: Threshold-based change detection\n- **Three-panel diff view**: Previous | Current | Difference (with red highlights)\n- **Direct image support**: Works with browser screenshots AND direct image URLs\n- **Visual selector support**: Compare specific page regions using CSS/XPath selectors\n- **Download images**: Download any of the three comparison images directly from the diff view\n\n## Performance\n\n- **OpenCV (default)**: 50-100x faster than SSIM\n- **Large screenshots**: Automatic downscaling for diff visualization (configurable via `MAX_DIFF_HEIGHT`/`MAX_DIFF_WIDTH`)\n- **Memory efficient**: Explicit cleanup of large objects for long-running processes\n- **JPEG diff images**: Smaller file sizes, faster rendering\n\n## How It Works\n\n1. **Fetch**: Screenshot captured via browser OR direct image URL fetched\n2. **MD5 Check**: Quick hash comparison - if identical, skip comparison\n3. **Region Selection** (optional): Crop to specific page region if visual selector is configured\n4. **OpenCV Comparison**: Fast pixel-level difference detection with Gaussian blur\n5. **Change Detection**: Percentage of changed pixels above threshold = change detected\n6. **Visualization**: Generate diff image with red-highlighted changed regions\n\n## Architecture\n\n### Default Method: OpenCV\n\nThe processor uses OpenCV's `cv2.absdiff()` for ultra-fast pixel-level comparison:\n\n```python\n# Convert to grayscale\ngray_from = cv2.cvtColor(image_from, cv2.COLOR_RGB2GRAY)\ngray_to = cv2.cvtColor(image_to, cv2.COLOR_RGB2GRAY)\n\n# Apply Gaussian blur (reduces noise, controlled by OPENCV_BLUR_SIGMA env var)\ngray_from = cv2.GaussianBlur(gray_from, (0, 0), sigma=0.8)\ngray_to = cv2.GaussianBlur(gray_to, (0, 0), sigma=0.8)\n\n# Calculate absolute difference\ndiff = cv2.absdiff(gray_from, gray_to)\n\n# Apply threshold (default: 30)\n_, thresh = cv2.threshold(diff, threshold, 255, cv2.THRESH_BINARY)\n\n# Count changed pixels\nchange_percentage = (changed_pixels / total_pixels) * 100\n```\n\n### Optional: Pixelmatch\n\nFor users who need better anti-aliasing detection (especially for text-heavy screenshots), **pixelmatch** can be optionally installed:\n\n```bash\npip install pybind11-pixelmatch>=0.1.3\n```\n\n**Note**: Pixelmatch uses a C++17 implementation via pybind11 and may have build issues on some platforms (particularly Alpine/musl systems with symbolic link security restrictions). The application will automatically fall back to OpenCV if pixelmatch is not available.\n\nTo use pixelmatch instead of OpenCV, set the environment variable:\n```bash\nCOMPARISON_METHOD=pixelmatch\n```\n\n#### When to use pixelmatch:\n- Screenshots with lots of text and anti-aliasing\n- Need to ignore minor font rendering differences between browser versions\n- 10-20x faster than SSIM (but slower than OpenCV)\n\n#### When to stick with OpenCV (default):\n- General webpage monitoring\n- Maximum performance (50-100x faster than SSIM)\n- Simple pixel-level change detection\n- Avoid build dependencies (Alpine/musl systems)\n\n## Configuration\n\n### Environment Variables\n\n```bash\n# Comparison method (opencv or pixelmatch)\nCOMPARISON_METHOD=opencv  # Default\n\n# OpenCV threshold (0-255, lower = more sensitive)\nCOMPARISON_THRESHOLD_OPENCV=30  # Default\n\n# Pixelmatch threshold (0-100, mapped to 0-1 scale)\nCOMPARISON_THRESHOLD_PIXELMATCH=10  # Default\n\n# Gaussian blur sigma for OpenCV (0 = no blur, higher = more blur)\nOPENCV_BLUR_SIGMA=0.8  # Default\n\n# Minimum change percentage to trigger detection\nOPENCV_MIN_CHANGE_PERCENT=0.1  # Default (0.1%)\nPIXELMATCH_MIN_CHANGE_PERCENT=0.1  # Default\n\n# Diff visualization image size limits (pixels)\nMAX_DIFF_HEIGHT=8000  # Default\nMAX_DIFF_WIDTH=900  # Default\n```\n\n### Per-Watch Configuration\n\n- **Comparison Threshold**: Can be configured per-watch in the edit form\n  - Very low sensitivity (10) - Only major changes\n  - Low sensitivity (20) - Significant changes\n  - Medium sensitivity (30) - Moderate changes (default)\n  - High sensitivity (50) - Small changes\n  - Very high sensitivity (75) - Any visible change\n\n### Visual Selector (Region Comparison)\n\nUse the \"Include filters\" field with CSS selectors or XPath to compare only specific page regions:\n\n```\n.content-area\n//div[@id='main']\n```\n\nThe processor will automatically crop both screenshots to the bounding box of the first matched element.\n\n## Dependencies\n\n### Required\n- `opencv-python-headless>=4.8.0.76` - Fast image comparison\n- `Pillow (PIL)` - Image loading and manipulation\n- `numpy` - Array operations\n\n### Optional\n- `pybind11-pixelmatch>=0.1.3` - Alternative comparison method with anti-aliasing detection\n\n## Change Detection Interpretation\n\n- **0%** = Identical images (or below minimum change threshold)\n- **0.1-1%** = Minor differences (anti-aliasing, slight rendering differences)\n- **1-5%** = Noticeable changes (text updates, small content changes)\n- **5-20%** = Significant changes (layout shifts, content additions)\n- **>20%** = Major differences (page redesign, large content changes)\n\n## Technical Notes\n\n### Memory Management\n```python\n# Explicit cleanup for long-running processes\nimg.close()  # Close PIL Images\nbuffer.close()  # Close BytesIO buffers\ndel large_array  # Mark numpy arrays for GC\n```\n\n### Diff Image Generation\n- Format: JPEG (quality=85, optimized)\n- Highlight: Red overlay (50% blend with original)\n- Auto-downscaling: Large screenshots downscaled for faster rendering\n- Base64 embedded: For direct template rendering\n\n### OpenCV Blur Parameters\nThe Gaussian blur reduces sensitivity to:\n- Font rendering differences\n- Anti-aliasing variations\n- JPEG compression artifacts\n- Minor pixel shifts (1-2 pixels)\n\nIncrease `OPENCV_BLUR_SIGMA` to make comparison more tolerant of these differences.\n\n## Comparison: OpenCV vs Pixelmatch vs SSIM\n\n| Feature | OpenCV | Pixelmatch | SSIM (old) |\n|---------|--------|------------|------------|\n| **Speed** | 50-100x faster | 10-20x faster | Baseline |\n| **Anti-aliasing** | Via blur | Built-in detection | Built-in |\n| **Text sensitivity** | High | Medium (AA-aware) | Medium |\n| **Dependencies** | opencv-python-headless | pybind11-pixelmatch + C++ compiler | scikit-image |\n| **Alpine/musl support** | ✅ Yes | ⚠️ Build issues | ✅ Yes |\n| **Memory usage** | Low | Low | High |\n| **Best for** | General use, max speed | Text-heavy screenshots | Deprecated |\n\n## Migration from SSIM\n\nIf you're upgrading from the old SSIM-based processor:\n\n1. **Thresholds are different**: SSIM used 0-1 scale (higher = more similar), OpenCV uses 0-255 pixel difference (lower = more similar)\n2. **Default threshold**: Start with 30 for OpenCV, adjust based on your needs\n3. **Performance**: Expect dramatically faster comparisons, especially for large screenshots\n4. **Accuracy**: OpenCV is more sensitive to pixel-level changes; increase `OPENCV_BLUR_SIGMA` if you're getting false positives\n\n## Future Enhancements\n\nPotential features for future consideration:\n\n- **Change region detection**: Highlight specific areas that changed with bounding boxes\n- **Perceptual hashing**: Pre-screening filter for even faster checks\n- **Ignore regions**: Exclude specific page areas (ads, timestamps) from comparison\n- **Text extraction**: OCR-based text comparison for semantic changes\n- **Adaptive thresholds**: Different sensitivity for different page regions\n\n## Resources\n\n- [OpenCV Documentation](https://docs.opencv.org/)\n- [pybind11-pixelmatch GitHub](https://github.com/whtsky/pybind11-pixelmatch)\n- [Pixelmatch (original JS library)](https://github.com/mapbox/pixelmatch)\n"
  },
  {
    "path": "changedetectionio/processors/image_ssim_diff/__init__.py",
    "content": "\"\"\"\nVisual/screenshot change detection using fast image comparison algorithms.\n\nThis processor compares screenshots using OpenCV (cv2.absdiff),\nwhich is 10-100x faster than SSIM while still detecting meaningful visual changes.\n\"\"\"\n\nimport os\nfrom pathlib import Path\n\nprocessor_description = \"Visual/Screenshot change detection (Fast)\"\nprocessor_name = \"image_ssim_diff\"\nprocessor_weight = 2  # Lower weight = appears at top, heavier weight = appears lower (bottom)\n\n# Processor capabilities\nsupports_visual_selector = True\nsupports_browser_steps = True\nsupports_text_filters_and_triggers = False\nsupports_text_filters_and_triggers_elements = False\nsupports_request_type = True\n\nPROCESSOR_CONFIG_NAME = f\"{Path(__file__).parent.name}.json\"\n\n# Subprocess timeout settings\n# Maximum time to wait for subprocess operations (seconds)\nPOLL_TIMEOUT_ABSOLUTE = int(os.getenv('OPENCV_SUBPROCESS_TIMEOUT', '20'))\n\n# Template tracking filename\nCROPPED_IMAGE_TEMPLATE_FILENAME = 'cropped_image_template.png'\n\nSCREENSHOT_COMPARISON_THRESHOLD_OPTIONS = [\n    ('200', 'Low sensitivity (only major changes)'),\n    ('80', 'Medium sensitivity (moderate changes - recommended)'),\n    ('20', 'High sensitivity (small changes)'),\n    ('0', 'Very high sensitivity (any change)')\n]\n\nSCREENSHOT_COMPARISON_THRESHOLD_OPTIONS_DEFAULT=0.999\nOPENCV_BLUR_SIGMA=float(os.getenv(\"OPENCV_BLUR_SIGMA\", \"3.0\"))\n"
  },
  {
    "path": "changedetectionio/processors/image_ssim_diff/difference.py",
    "content": "\"\"\"\nScreenshot diff visualization for fast image comparison processor.\n\nAll image operations now use ImageDiffHandler abstraction for clean separation\nof concerns and easy backend swapping (LibVIPS, OpenCV, PIL, etc.).\n\"\"\"\n\nimport os\nimport json\nimport time\nfrom flask_babel import gettext\nfrom loguru import logger\n\nfrom changedetectionio.processors.image_ssim_diff import SCREENSHOT_COMPARISON_THRESHOLD_OPTIONS_DEFAULT, PROCESSOR_CONFIG_NAME, \\\n    OPENCV_BLUR_SIGMA\n\n# All image operations now use OpenCV via isolated_opencv subprocess handler\n# No direct handler imports needed - subprocess isolation handles everything\n\n# Maximum dimensions for diff visualization (can be overridden via environment variable)\n# Large screenshots don't need full resolution for visual inspection\n# Reduced defaults to minimize memory usage - 2000px height is plenty for diff viewing\nMAX_DIFF_HEIGHT = int(os.getenv('MAX_DIFF_HEIGHT', '8000'))\nMAX_DIFF_WIDTH = int(os.getenv('MAX_DIFF_WIDTH', '900'))\n\n\ndef get_asset(asset_name, watch, datastore, request):\n    \"\"\"\n    Get processor-specific binary assets for streaming.\n\n    Uses ImageDiffHandler for all image operations - no more multiprocessing needed\n    as LibVIPS handles threading/memory internally.\n\n    Supported assets:\n    - 'before': The previous/from screenshot\n    - 'after': The current/to screenshot\n    - 'rendered_diff': The generated diff visualization with red highlights\n\n    Args:\n        asset_name: Name of the asset to retrieve ('before', 'after', 'rendered_diff')\n        watch: Watch object\n        datastore: Datastore object\n        request: Flask request (for from_version/to_version query params)\n\n    Returns:\n        tuple: (binary_data, content_type, cache_control_header) or None if not found\n    \"\"\"\n    # Get version parameters from query string\n    versions = list(watch.history.keys())\n\n    if len(versions) < 2:\n        return None\n\n    from_version = request.args.get('from_version', versions[-2] if len(versions) >= 2 else versions[0])\n    to_version = request.args.get('to_version', versions[-1])\n\n    # Validate versions exist\n    if from_version not in versions:\n        from_version = versions[-2] if len(versions) >= 2 else versions[0]\n    if to_version not in versions:\n        to_version = versions[-1]\n\n    try:\n        if asset_name == 'before':\n            # Return the 'from' screenshot with bounding box if configured\n            img_bytes = watch.get_history_snapshot(timestamp=from_version)\n            img_bytes = _draw_bounding_box_if_configured(img_bytes, watch, datastore)\n            mime_type = _detect_mime_type(img_bytes)\n            return (img_bytes, mime_type, 'public, max-age=3600')\n\n        elif asset_name == 'after':\n            # Return the 'to' screenshot with bounding box if configured\n            img_bytes = watch.get_history_snapshot(timestamp=to_version)\n            img_bytes = _draw_bounding_box_if_configured(img_bytes, watch, datastore)\n            mime_type = _detect_mime_type(img_bytes)\n            return (img_bytes, mime_type, 'public, max-age=3600')\n\n        elif asset_name == 'rendered_diff':\n            # Generate diff in isolated subprocess to prevent memory leaks\n            # Subprocess provides complete memory isolation\n            from .image_handler import isolated_opencv as process_screenshot_handler\n\n            img_bytes_from = watch.get_history_snapshot(timestamp=from_version)\n            img_bytes_to = watch.get_history_snapshot(timestamp=to_version)\n\n            # Get pixel difference threshold sensitivity (per-watch > global)\n            # This controls how different a pixel must be (0-255 scale) to count as \"changed\"\n            from changedetectionio import processors\n            processor_instance = processors.difference_detection_processor(datastore, watch.get('uuid'))\n            processor_config = processor_instance.get_extra_watch_config(PROCESSOR_CONFIG_NAME)\n\n            pixel_difference_threshold_sensitivity = processor_config.get('pixel_difference_threshold_sensitivity')\n            if not pixel_difference_threshold_sensitivity:\n                pixel_difference_threshold_sensitivity = datastore.data['settings']['application'].get(\n                    'pixel_difference_threshold_sensitivity', SCREENSHOT_COMPARISON_THRESHOLD_OPTIONS_DEFAULT)\n            try:\n                pixel_difference_threshold_sensitivity = int(pixel_difference_threshold_sensitivity)\n            except (ValueError, TypeError):\n                logger.warning(\n                    f\"Invalid pixel_difference_threshold_sensitivity value '{pixel_difference_threshold_sensitivity}', using default\")\n                pixel_difference_threshold_sensitivity = SCREENSHOT_COMPARISON_THRESHOLD_OPTIONS_DEFAULT\n\n            logger.debug(f\"Pixel difference threshold sensitivity is {pixel_difference_threshold_sensitivity}\")\n\n\n            # Generate diff in isolated subprocess (async-safe)\n            import asyncio\n            import threading\n\n            # Async-safe wrapper: runs coroutine in new thread with its own event loop\n            def run_async_in_thread():\n                return asyncio.run(\n                    process_screenshot_handler.generate_diff_isolated(\n                        img_bytes_from,\n                        img_bytes_to,\n                        pixel_difference_threshold=int(pixel_difference_threshold_sensitivity),\n                        blur_sigma=OPENCV_BLUR_SIGMA,\n                        max_width=MAX_DIFF_WIDTH,\n                        max_height=MAX_DIFF_HEIGHT\n                    )\n                )\n\n            # Run in thread to avoid blocking event loop if called from async context\n            result_container = [None]\n            exception_container = [None]\n\n            def thread_target():\n                try:\n                    result_container[0] = run_async_in_thread()\n                except Exception as e:\n                    exception_container[0] = e\n\n            thread = threading.Thread(target=thread_target, daemon=True, name=\"ImageDiff-Asset\")\n            thread.start()\n            thread.join(timeout=60)\n\n            if exception_container[0]:\n                raise exception_container[0]\n\n            diff_image_bytes = result_container[0]\n\n            if diff_image_bytes:\n                # Note: Bounding box drawing on diff not yet implemented\n                return (diff_image_bytes, 'image/jpeg', 'public, max-age=300')\n            else:\n                logger.error(\"Failed to generate diff in subprocess\")\n                return None\n\n        else:\n            # Unknown asset\n            return None\n\n    except Exception as e:\n        logger.error(f\"Failed to get asset '{asset_name}': {e}\")\n        import traceback\n        logger.error(traceback.format_exc())\n        return None\n\n\ndef _detect_mime_type(img_bytes):\n    \"\"\"\n    Detect MIME type using puremagic (same as Watch.py).\n\n    Args:\n        img_bytes: Image bytes\n\n    Returns:\n        str: MIME type (e.g., 'image/png', 'image/jpeg')\n    \"\"\"\n    try:\n        import puremagic\n        detections = puremagic.magic_string(img_bytes[:2048])\n        if detections:\n            mime_type = detections[0].mime_type\n            logger.trace(f\"Detected MIME type: {mime_type}\")\n            return mime_type\n        else:\n            logger.trace(\"No MIME type detected, using 'image/png' fallback\")\n            return 'image/png'\n    except Exception as e:\n        logger.warning(f\"puremagic detection failed: {e}, using 'image/png' fallback\")\n        return 'image/png'\n\n\ndef _draw_bounding_box_if_configured(img_bytes, watch, datastore):\n    \"\"\"\n    Draw blue bounding box on image if configured in processor settings.\n    Uses isolated subprocess to prevent memory leaks from large images.\n\n    Supports two modes:\n    - \"Select by element\": Use include_filter to find xpath element bbox\n    - \"Draw area\": Use manually drawn bounding_box from config\n\n    Args:\n        img_bytes: Image bytes (PNG)\n        watch: Watch object\n        datastore: Datastore object\n\n    Returns:\n        Image bytes (possibly with bounding box drawn)\n    \"\"\"\n    try:\n        # Get processor configuration\n        from changedetectionio import processors\n        processor_instance = processors.difference_detection_processor(datastore, watch.get('uuid'))\n        processor_name = watch.get('processor', 'default')\n        config_filename = f'{processor_name}.json'\n        processor_config = processor_instance.get_extra_watch_config(config_filename)\n\n        if not processor_config:\n            return img_bytes\n\n        selection_mode = processor_config.get('selection_mode', 'draw')\n        x, y, width, height = None, None, None, None\n\n        # Mode 1: Select by element (use include_filter + xpath_data)\n        if selection_mode == 'element':\n            include_filters = watch.get('include_filters', [])\n\n            if include_filters and len(include_filters) > 0:\n                first_filter = include_filters[0].strip()\n\n                # Get xpath_data from watch history\n                history_keys = list(watch.history.keys())\n                if history_keys:\n                    latest_snapshot = watch.get_history_snapshot(timestamp=history_keys[-1])\n                    xpath_data_path = watch.get_xpath_data_filepath(timestamp=history_keys[-1])\n\n                    try:\n                        import gzip\n                        with gzip.open(xpath_data_path, 'rt') as f:\n                            xpath_data = json.load(f)\n\n                        # Find matching element\n                        for element in xpath_data.get('size_pos', []):\n                            if element.get('xpath') == first_filter and element.get('highlight_as_custom_filter'):\n                                x = element.get('left', 0)\n                                y = element.get('top', 0)\n                                width = element.get('width', 0)\n                                height = element.get('height', 0)\n                                logger.debug(f\"Found element bbox for filter '{first_filter}': x={x}, y={y}, w={width}, h={height}\")\n                                break\n                    except Exception as e:\n                        logger.warning(f\"Failed to load xpath_data for element selection: {e}\")\n\n        # Mode 2: Draw area (use manually configured bbox)\n        else:\n            bounding_box = processor_config.get('bounding_box')\n            if bounding_box:\n                # Parse bounding box: \"x,y,width,height\"\n                parts = [int(p.strip()) for p in bounding_box.split(',')]\n                if len(parts) == 4:\n                    x, y, width, height = parts\n                else:\n                    logger.warning(f\"Invalid bounding box format: {bounding_box}\")\n\n        # If no bbox found, return original image\n        if x is None or y is None or width is None or height is None:\n            return img_bytes\n\n        # Use isolated subprocess to prevent memory leaks from large images\n        from .image_handler import isolated_opencv\n        import asyncio\n        import threading\n\n        # Async-safe wrapper: runs coroutine in new thread with its own event loop\n        # This prevents blocking when called from async context (update worker)\n        def run_async_in_thread():\n            return asyncio.run(\n                isolated_opencv.draw_bounding_box_isolated(\n                    img_bytes, x, y, width, height,\n                    color=(255, 0, 0),  # Blue in BGR format\n                    thickness=3\n                )\n            )\n\n        # Always run in thread to avoid blocking event loop if called from async context\n        result_container = [None]\n        exception_container = [None]\n\n        def thread_target():\n            try:\n                result_container[0] = run_async_in_thread()\n            except Exception as e:\n                exception_container[0] = e\n\n        thread = threading.Thread(target=thread_target, daemon=True, name=\"ImageDiff-BoundingBox\")\n        thread.start()\n        thread.join(timeout=15)\n\n        if exception_container[0]:\n            raise exception_container[0]\n\n        result = result_container[0]\n\n        # Return result or original if subprocess failed\n        return result if result else img_bytes\n\n    except Exception as e:\n        logger.warning(f\"Failed to draw bounding box: {e}\")\n        import traceback\n        logger.debug(traceback.format_exc())\n        return img_bytes\n\n\ndef render(watch, datastore, request, url_for, render_template, flash, redirect):\n    \"\"\"\n    Render the screenshot comparison diff page.\n\n    Uses ImageDiffHandler for all image operations.\n\n    Args:\n        watch: Watch object\n        datastore: Datastore object\n        request: Flask request\n        url_for: Flask url_for function\n        render_template: Flask render_template function\n        flash: Flask flash function\n        redirect: Flask redirect function\n\n    Returns:\n        Rendered template or redirect\n    \"\"\"\n    # Get version parameters (from_version, to_version)\n    versions = list(watch.history.keys())\n\n    if len(versions) < 2:\n        flash(gettext(\"Not enough history to compare. Need at least 2 snapshots.\"), \"error\")\n        return redirect(url_for('watchlist.index'))\n\n    # Default: compare latest two versions\n    from_version = request.args.get('from_version', versions[-2] if len(versions) >= 2 else versions[0])\n    to_version = request.args.get('to_version', versions[-1])\n\n    # Validate versions exist\n    if from_version not in versions:\n        from_version = versions[-2] if len(versions) >= 2 else versions[0]\n    if to_version not in versions:\n        to_version = versions[-1]\n\n    # Get pixel difference threshold sensitivity (per-watch > global > env default)\n    pixel_difference_threshold_sensitivity = watch.get('pixel_difference_threshold_sensitivity')\n    if not pixel_difference_threshold_sensitivity or pixel_difference_threshold_sensitivity == '':\n        pixel_difference_threshold_sensitivity = datastore.data['settings']['application'].get('pixel_difference_threshold_sensitivity', SCREENSHOT_COMPARISON_THRESHOLD_OPTIONS_DEFAULT)\n\n    # Convert to appropriate type\n    try:\n        pixel_difference_threshold_sensitivity = float(pixel_difference_threshold_sensitivity)\n    except (ValueError, TypeError):\n        logger.warning(f\"Invalid pixel_difference_threshold_sensitivity value '{pixel_difference_threshold_sensitivity}', using default\")\n        pixel_difference_threshold_sensitivity = 30.0\n\n    # Get blur sigma\n    blur_sigma = OPENCV_BLUR_SIGMA\n\n    # Load screenshots from history\n    try:\n        img_bytes_from = watch.get_history_snapshot(timestamp=from_version)\n        img_bytes_to = watch.get_history_snapshot(timestamp=to_version)\n\n    except Exception as e:\n        logger.error(f\"Failed to load screenshots: {e}\")\n        flash(gettext(\"Failed to load screenshots: {}\").format(e), \"error\")\n        return redirect(url_for('watchlist.index'))\n\n    # Calculate change percentage using isolated subprocess to prevent memory leaks (async-safe)\n    now = time.time()\n    try:\n        from .image_handler import isolated_opencv as process_screenshot_handler\n        import asyncio\n        import threading\n\n        # Async-safe wrapper: runs coroutine in new thread with its own event loop\n        def run_async_in_thread():\n            return asyncio.run(\n                process_screenshot_handler.calculate_change_percentage_isolated(\n                    img_bytes_from,\n                    img_bytes_to,\n                    pixel_difference_threshold=int(pixel_difference_threshold_sensitivity),\n                    blur_sigma=blur_sigma,\n                    max_width=MAX_DIFF_WIDTH,\n                    max_height=MAX_DIFF_HEIGHT\n                )\n            )\n\n        # Run in thread to avoid blocking event loop if called from async context\n        result_container = [None]\n        exception_container = [None]\n\n        def thread_target():\n            try:\n                result_container[0] = run_async_in_thread()\n            except Exception as e:\n                exception_container[0] = e\n\n        thread = threading.Thread(target=thread_target, daemon=True, name=\"ImageDiff-ChangePercentage\")\n        thread.start()\n        thread.join(timeout=60)\n\n        if exception_container[0]:\n            raise exception_container[0]\n\n        change_percentage = result_container[0]\n\n        method_display = f\"{process_screenshot_handler.IMPLEMENTATION_NAME} (pixel_diff_threshold: {pixel_difference_threshold_sensitivity:.0f})\"\n        logger.debug(f\"Done change percentage calculation in {time.time() - now:.2f}s\")\n\n    except Exception as e:\n        logger.error(f\"Failed to calculate change percentage: {e}\")\n        import traceback\n        logger.error(traceback.format_exc())\n        flash(gettext(\"Failed to calculate diff: {}\").format(e), \"error\")\n        return redirect(url_for('watchlist.index'))\n\n    # Load historical data if available (for charts/visualization)\n    comparison_data = {}\n    comparison_config_path = os.path.join(watch.data_dir, \"visual_comparison_data.json\")\n    if os.path.isfile(comparison_config_path):\n        try:\n            with open(comparison_config_path, 'r') as f:\n                comparison_data = json.load(f)\n        except Exception as e:\n            logger.warning(f\"Failed to load comparison history data: {e}\")\n\n    # Render custom template\n    # Template path is namespaced to avoid conflicts with other processors\n    # Images are now served via separate /processor-asset/ endpoints instead of base64\n    return render_template(\n        'image_ssim_diff/diff.html',\n        change_percentage=change_percentage,\n        comparison_data=comparison_data,  # Full history for charts/visualization\n        comparison_method=method_display,\n        current_diff_url=watch['url'],\n        from_version=from_version,\n        percentage_different=change_percentage,\n        threshold=pixel_difference_threshold_sensitivity,\n        to_version=to_version,\n        uuid=watch.get('uuid'),\n        versions=versions,\n        watch=watch,\n    )\n"
  },
  {
    "path": "changedetectionio/processors/image_ssim_diff/edit_hook.py",
    "content": "\"\"\"\nOptional hook called when processor settings are saved in edit page.\n\nThis hook analyzes the selected region to determine if template matching\nshould be enabled for tracking content movement.\n\nTemplate matching is controlled via ENABLE_TEMPLATE_TRACKING env var (default: False).\n\"\"\"\n\nimport io\nimport os\nfrom loguru import logger\nfrom changedetectionio import strtobool\nfrom . import CROPPED_IMAGE_TEMPLATE_FILENAME\n\n# Template matching controlled via environment variable (default: disabled)\n# Set ENABLE_TEMPLATE_TRACKING=True to enable\nTEMPLATE_MATCHING_ENABLED = strtobool(os.getenv('ENABLE_TEMPLATE_TRACKING', 'False'))\nIMPORT_ERROR = \"Template matching disabled (set ENABLE_TEMPLATE_TRACKING=True to enable)\"\n\n\ndef on_config_save(watch, processor_config, datastore):\n    \"\"\"\n    Called after processor config is saved in edit page.\n\n    Analyzes the bounding box region to determine if it has enough\n    visual features (texture/edges) to enable template matching for\n    tracking content movement when page layout shifts.\n\n    Args:\n        watch: Watch object\n        processor_config: Dict of processor-specific config\n        datastore: Datastore object\n\n    Returns:\n        dict: Updated processor_config with auto_track_region setting\n    \"\"\"\n    # Check if template matching is globally enabled via ENV var\n    if not TEMPLATE_MATCHING_ENABLED:\n        logger.debug(\"Template tracking disabled via ENABLE_TEMPLATE_TRACKING env var\")\n        processor_config['auto_track_region'] = False\n        return processor_config\n\n    bounding_box = processor_config.get('bounding_box')\n\n    if not bounding_box:\n        # No bounding box, disable tracking\n        processor_config['auto_track_region'] = False\n        logger.debug(\"No bounding box set, disabled auto-tracking\")\n        return processor_config\n\n    try:\n        # Get the latest screenshot from watch history\n        history_keys = list(watch.history.keys())\n        if len(history_keys) == 0:\n            logger.warning(\"No screenshot history available yet, cannot analyze for tracking\")\n            processor_config['auto_track_region'] = False\n            return processor_config\n\n        # Get latest screenshot\n        latest_timestamp = history_keys[-1]\n        screenshot_bytes = watch.get_history_snapshot(timestamp=latest_timestamp)\n\n        if not screenshot_bytes:\n            logger.warning(\"Could not load screenshot for analysis\")\n            processor_config['auto_track_region'] = False\n            return processor_config\n\n        # Parse bounding box\n        parts = [int(p.strip()) for p in bounding_box.split(',')]\n        if len(parts) != 4:\n            logger.warning(\"Invalid bounding box format\")\n            processor_config['auto_track_region'] = False\n            return processor_config\n\n        x, y, width, height = parts\n\n        # Analyze the region for features/texture\n        has_enough_features = analyze_region_features(screenshot_bytes, x, y, width, height)\n\n        if has_enough_features:\n            logger.info(f\"Region has sufficient features for tracking - enabling auto_track_region\")\n            processor_config['auto_track_region'] = True\n\n            # Save the template as cropped.jpg in watch data directory\n            save_template_to_file(watch, screenshot_bytes, x, y, width, height)\n\n        else:\n            logger.info(f\"Region lacks distinctive features - disabling auto_track_region\")\n            processor_config['auto_track_region'] = False\n\n            # Remove old template file if exists\n            template_path = os.path.join(watch.data_dir, CROPPED_IMAGE_TEMPLATE_FILENAME)\n            if os.path.exists(template_path):\n                os.remove(template_path)\n                logger.debug(f\"Removed old template file: {template_path}\")\n\n        return processor_config\n\n    except Exception as e:\n        logger.error(f\"Error analyzing region for tracking: {e}\")\n        processor_config['auto_track_region'] = False\n        return processor_config\n\n\ndef analyze_region_features(screenshot_bytes, x, y, width, height):\n    \"\"\"\n    Analyze if a region has enough visual features for template matching.\n\n    Uses OpenCV to detect corners/edges. If the region has distinctive\n    features, template matching can reliably track it when it moves.\n\n    Args:\n        screenshot_bytes: Full screenshot as bytes\n        x, y, width, height: Bounding box coordinates\n\n    Returns:\n        bool: True if region has enough features, False otherwise\n    \"\"\"\n    # Template matching disabled - would need OpenCV implementation for region analysis\n    if not TEMPLATE_MATCHING_ENABLED:\n        logger.warning(f\"Cannot analyze region features: {IMPORT_ERROR}\")\n        return False\n\n    # Note: Original implementation used LibVIPS handler to crop region, then OpenCV\n    # for feature detection (goodFeaturesToTrack, Canny edge detection, variance).\n    # If re-implementing, use OpenCV directly for both cropping and analysis.\n    # Feature detection would use: cv2.goodFeaturesToTrack, cv2.Canny, np.var\n    return False\n\n\ndef save_template_to_file(watch, screenshot_bytes, x, y, width, height):\n    \"\"\"\n    Extract the template region and save as cropped_image_template.png in watch data directory.\n\n    This is a convenience wrapper around handler.save_template() that handles\n    watch directory setup and path construction.\n\n    Args:\n        watch: Watch object\n        screenshot_bytes: Full screenshot as bytes\n        x, y, width, height: Bounding box coordinates\n    \"\"\"\n    # Template matching disabled - would need OpenCV implementation for template saving\n    if not TEMPLATE_MATCHING_ENABLED:\n        logger.warning(f\"Cannot save template: {IMPORT_ERROR}\")\n        return\n\n    # Note: Original implementation used LibVIPS handler to crop and save region.\n    # If re-implementing, use OpenCV (cv2.imdecode, crop with array slicing, cv2.imwrite).\n    return\n"
  },
  {
    "path": "changedetectionio/processors/image_ssim_diff/forms.py",
    "content": "\"\"\"\nConfiguration forms for fast screenshot comparison processor.\n\"\"\"\n\nfrom wtforms import SelectField, StringField, validators, ValidationError, IntegerField\nfrom flask_babel import lazy_gettext as _l\nfrom changedetectionio.forms import processor_text_json_diff_form\nimport re\n\nfrom changedetectionio.processors.image_ssim_diff import SCREENSHOT_COMPARISON_THRESHOLD_OPTIONS\n\n\ndef validate_bounding_box(form, field):\n    \"\"\"Validate bounding box format: x,y,width,height with integers.\"\"\"\n    if not field.data:\n        return  # Optional field\n\n    if len(field.data) > 100:\n        raise ValidationError(_l('Bounding box value is too long'))\n\n    # Should be comma-separated integers\n    if not re.match(r'^\\d+,\\d+,\\d+,\\d+$', field.data):\n        raise ValidationError(_l('Bounding box must be in format: x,y,width,height (integers only)'))\n\n    # Validate values are reasonable (not negative, not ridiculously large)\n    parts = [int(p) for p in field.data.split(',')]\n    for part in parts:\n        if part < 0:\n            raise ValidationError(_l('Bounding box values must be non-negative'))\n        if part > 10000:  # Reasonable max screen dimension\n            raise ValidationError(_l('Bounding box values are too large'))\n\n\ndef validate_selection_mode(form, field):\n    \"\"\"Validate selection mode value.\"\"\"\n    if not field.data:\n        return  # Optional field\n\n    if field.data not in ['element', 'draw']:\n        raise ValidationError(_l('Selection mode must be either \"element\" or \"draw\"'))\n\n\nclass processor_settings_form(processor_text_json_diff_form):\n    \"\"\"Form for fast image comparison processor settings.\"\"\"\n\n    processor_config_min_change_percentage = IntegerField(\n        _l('Minimum Change Percentage'),\n        validators=[\n            validators.Optional(),\n            validators.NumberRange(min=1, max=100, message=_l('Must be between 0 and 100'))\n        ],\n        render_kw={\"placeholder\": \"Use global default (0.1)\"}\n    )\n\n    processor_config_pixel_difference_threshold_sensitivity = SelectField(\n        _l('Pixel Difference Sensitivity'),\n        choices=[\n                    ('', _l('Use global default'))\n                ] + SCREENSHOT_COMPARISON_THRESHOLD_OPTIONS,\n        validators=[validators.Optional()],\n        default=''\n    )\n\n    # Processor-specific config fields (stored in separate JSON file)\n    processor_config_bounding_box = StringField(\n        _l('Bounding Box'),\n        validators=[\n            validators.Optional(),\n            validators.Length(max=100, message=_l('Bounding box value is too long')),\n            validate_bounding_box\n        ],\n        render_kw={\"style\": \"display: none;\", \"id\": \"bounding_box\"}\n    )\n\n    processor_config_selection_mode = StringField(\n        _l('Selection Mode'),\n        validators=[\n            validators.Optional(),\n            validators.Length(max=20, message=_l('Selection mode value is too long')),\n            validate_selection_mode\n        ],\n        render_kw={\"style\": \"display: none;\", \"id\": \"selection_mode\"}\n    )\n\n    def extra_tab_content(self):\n        \"\"\"Tab label for processor-specific settings.\"\"\"\n        return _l('Screenshot Comparison')\n\n    def extra_form_content(self):\n        \"\"\"Render processor-specific form fields.\n        @NOTE: prepend processor_config_* to the field name so it will save into its own datadir/uuid/image_ssim_diff.json and be read at process time\n        \"\"\"\n        return '''\n        {% from '_helpers.html' import render_field %}\n        
\n            Screenshot Comparison Settings\n\n            
\n                {{ render_field(form.processor_config_min_change_percentage) }}\n                \n                    What percentage of pixels must change to trigger a detection?
\n                    For example, 0.1% means if 0.1% or more of the pixels change, it counts as a change.
\n                    Lower values = more sensitive (detect smaller changes).
\n                    Higher values = less sensitive (only detect larger changes).
\n                    Leave blank to use global default (0.1%).\n                \n            
\n\n            
\n                {{ render_field(form.processor_config_pixel_difference_threshold_sensitivity) }}\n                \n                    How different must an individual pixel be to count as \"changed\"?
\n                    Low sensitivity (75) = Only count pixels that changed significantly (0-255 scale).
\n                    High sensitivity (20) = Count pixels with small changes as different.
\n                    Very high (0) = Any pixel change counts.
\n                    Select \"Use global default\" to inherit the system-wide setting.\n                \n            
\n        
\n        '''\n"
  },
  {
    "path": "changedetectionio/processors/image_ssim_diff/image_handler/__init__.py",
    "content": "\"\"\"\nAbstract base class for image processing operations.\n\nAll image operations for the image_ssim_diff processor must be implemented\nthrough this interface to allow different backends (libvips, OpenCV, PIL, etc.).\n\"\"\"\n\nfrom abc import ABC, abstractmethod\nfrom typing import Tuple, Optional, Any\n\n\nclass ImageDiffHandler(ABC):\n    \"\"\"\n    Abstract base class for image processing operations.\n\n    Implementations must handle all image operations needed for screenshot\n    comparison including loading, cropping, resizing, diffing, and overlays.\n    \"\"\"\n\n    @abstractmethod\n    def load_from_bytes(self, img_bytes: bytes) -> Any:\n        \"\"\"\n        Load image from bytes.\n\n        Args:\n            img_bytes: Image data as bytes (PNG, JPEG, etc.)\n\n        Returns:\n            Handler-specific image object\n        \"\"\"\n        pass\n\n    @abstractmethod\n    def save_to_bytes(self, img: Any, format: str = 'png', quality: int = 85) -> bytes:\n        \"\"\"\n        Save image to bytes.\n\n        Args:\n            img: Handler-specific image object\n            format: Output format ('png' or 'jpeg')\n            quality: Quality for JPEG (1-100)\n\n        Returns:\n            Image data as bytes\n        \"\"\"\n        pass\n\n    @abstractmethod\n    def crop(self, img: Any, left: int, top: int, right: int, bottom: int) -> Any:\n        \"\"\"\n        Crop image to specified region.\n\n        Args:\n            img: Handler-specific image object\n            left: Left coordinate\n            top: Top coordinate\n            right: Right coordinate\n            bottom: Bottom coordinate\n\n        Returns:\n            Cropped image object\n        \"\"\"\n        pass\n\n    @abstractmethod\n    def resize(self, img: Any, max_width: int, max_height: int) -> Any:\n        \"\"\"\n        Resize image maintaining aspect ratio.\n\n        Args:\n            img: Handler-specific image object\n            max_width: Maximum width in pixels\n            max_height: Maximum height in pixels\n\n        Returns:\n            Resized image object\n        \"\"\"\n        pass\n\n    @abstractmethod\n    def get_dimensions(self, img: Any) -> Tuple[int, int]:\n        \"\"\"\n        Get image dimensions.\n\n        Args:\n            img: Handler-specific image object\n\n        Returns:\n            Tuple of (width, height)\n        \"\"\"\n        pass\n\n    @abstractmethod\n    def to_grayscale(self, img: Any) -> Any:\n        \"\"\"\n        Convert image to grayscale.\n\n        Args:\n            img: Handler-specific image object\n\n        Returns:\n            Grayscale image object\n        \"\"\"\n        pass\n\n    @abstractmethod\n    def gaussian_blur(self, img: Any, sigma: float) -> Any:\n        \"\"\"\n        Apply Gaussian blur to image.\n\n        Args:\n            img: Handler-specific image object\n            sigma: Blur sigma value (0 = no blur)\n\n        Returns:\n            Blurred image object\n        \"\"\"\n        pass\n\n    @abstractmethod\n    def absolute_difference(self, img1: Any, img2: Any) -> Any:\n        \"\"\"\n        Calculate absolute difference between two images.\n\n        Args:\n            img1: First image (handler-specific object)\n            img2: Second image (handler-specific object)\n\n        Returns:\n            Difference image object\n        \"\"\"\n        pass\n\n    @abstractmethod\n    def threshold(self, img: Any, threshold_value: int) -> Tuple[float, Any]:\n        \"\"\"\n        Apply threshold to image and calculate change percentage.\n\n        Args:\n            img: Handler-specific image object (typically grayscale difference)\n            threshold_value: Threshold value (0-255)\n\n        Returns:\n            Tuple of (change_percentage, binary_mask)\n            - change_percentage: Percentage of pixels above threshold (0-100)\n            - binary_mask: Handler-specific binary mask object\n        \"\"\"\n        pass\n\n    @abstractmethod\n    def apply_red_overlay(self, img: Any, mask: Any) -> bytes:\n        \"\"\"\n        Apply red overlay to image where mask is True.\n\n        Args:\n            img: Handler-specific image object (color)\n            mask: Handler-specific binary mask object\n\n        Returns:\n            JPEG bytes with red overlay applied\n        \"\"\"\n        pass\n\n    @abstractmethod\n    def close(self, img: Any) -> None:\n        \"\"\"\n        Clean up image resources if needed.\n\n        Args:\n            img: Handler-specific image object\n        \"\"\"\n        pass\n\n    @abstractmethod\n    def find_template(\n        self,\n        img: Any,\n        template_img: Any,\n        original_bbox: Tuple[int, int, int, int],\n        search_tolerance: float = 0.2\n    ) -> Optional[Tuple[int, int, int, int]]:\n        \"\"\"\n        Find template in image using template matching.\n\n        Args:\n            img: Handler-specific image object to search in\n            template_img: Handler-specific template image object to find\n            original_bbox: Original bounding box (left, top, right, bottom)\n            search_tolerance: How far to search (0.2 = ±20% of region size)\n\n        Returns:\n            New bounding box (left, top, right, bottom) or None if not found\n        \"\"\"\n        pass\n\n    @abstractmethod\n    def save_template(\n        self,\n        img: Any,\n        bbox: Tuple[int, int, int, int],\n        output_path: str\n    ) -> bool:\n        \"\"\"\n        Save a cropped region as a template file.\n\n        Args:\n            img: Handler-specific image object\n            bbox: Bounding box to crop (left, top, right, bottom)\n            output_path: Where to save the template PNG\n\n        Returns:\n            True if successful, False otherwise\n        \"\"\"\n        pass\n\n    @abstractmethod\n    def draw_bounding_box(\n        self,\n        img_bytes: bytes,\n        x: int,\n        y: int,\n        width: int,\n        height: int,\n        color: Tuple[int, int, int] = (255, 0, 0),\n        thickness: int = 3\n    ) -> bytes:\n        \"\"\"\n        Draw a bounding box rectangle on image.\n\n        Args:\n            img_bytes: Image data as bytes\n            x: Left coordinate\n            y: Top coordinate\n            width: Box width\n            height: Box height\n            color: BGR color tuple (default: blue)\n            thickness: Line thickness in pixels\n\n        Returns:\n            Image bytes with bounding box drawn\n        \"\"\"\n        pass\n"
  },
  {
    "path": "changedetectionio/processors/image_ssim_diff/image_handler/isolated_libvips.py",
    "content": "\"\"\"\nSubprocess-isolated image operations for memory leak prevention.\n\nLibVIPS accumulates C-level memory in long-running processes that cannot be\nreclaimed by Python's GC or libvips cache management. Using subprocess isolation\nensures complete memory cleanup when the process exits.\n\nThis module wraps LibvipsImageDiffHandler operations in multiprocessing for\ncomplete memory isolation without code duplication.\n\nResearch: https://github.com/libvips/pyvips/issues/234\n\"\"\"\n\nimport multiprocessing\n\n# CRITICAL: Use 'spawn' context instead of 'fork' to avoid inheriting parent's\n# LibVIPS threading state which can cause hangs in gaussblur operations\n# https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods\n\n\ndef _worker_generate_diff(conn, img_bytes_from, img_bytes_to, threshold, blur_sigma, max_width, max_height):\n    \"\"\"\n    Worker: Generate diff visualization using LibvipsImageDiffHandler in isolated subprocess.\n\n    This runs in a separate process for complete memory isolation.\n    Uses print() instead of loguru to avoid forking issues.\n    \"\"\"\n    try:\n        # Import handler inside worker\n        from .libvips_handler import LibvipsImageDiffHandler\n\n        print(f\"[Worker] Initializing handler\", flush=True)\n        handler = LibvipsImageDiffHandler()\n\n        # Load images using handler\n        img_from = handler.load_from_bytes(img_bytes_from)\n        img_to = handler.load_from_bytes(img_bytes_to)\n\n        # Ensure same size\n        w1, h1 = handler.get_dimensions(img_from)\n        w2, h2 = handler.get_dimensions(img_to)\n        if (w1, h1) != (w2, h2):\n            img_from = handler.resize(img_from, w2, h2)\n\n        # Downscale for faster diff visualization\n        img_from = handler.resize(img_from, max_width, max_height)\n        img_to = handler.resize(img_to, max_width, max_height)\n\n        # Convert to grayscale\n        gray_from = handler.to_grayscale(img_from)\n        gray_to = handler.to_grayscale(img_to)\n\n        # Optional blur - DISABLED due to LibVIPS threading issues in fork\n        # gray_from = handler.gaussian_blur(gray_from, blur_sigma)\n        # gray_to = handler.gaussian_blur(gray_to, blur_sigma)\n\n        # Calculate difference\n        diff = handler.absolute_difference(gray_from, gray_to)\n\n        # Threshold to get mask\n        _, diff_mask = handler.threshold(diff, int(threshold))\n\n        # Generate diff image with red overlay\n        diff_image_bytes = handler.apply_red_overlay(img_to, diff_mask)\n\n        print(f\"[Worker] Generated diff ({len(diff_image_bytes)} bytes)\", flush=True)\n        conn.send(diff_image_bytes)\n\n    except Exception as e:\n        print(f\"[Worker] Error: {e}\", flush=True)\n        import traceback\n        traceback.print_exc()\n        conn.send(None)\n    finally:\n        conn.close()\n\n\ndef generate_diff_isolated(img_bytes_from, img_bytes_to, threshold, blur_sigma, max_width, max_height):\n    \"\"\"\n    Generate diff visualization in isolated subprocess for memory leak prevention.\n\n    Args:\n        img_bytes_from: Previous screenshot bytes\n        img_bytes_to: Current screenshot bytes\n        threshold: Pixel difference threshold\n        blur_sigma: Gaussian blur sigma\n        max_width: Maximum width for diff\n        max_height: Maximum height for diff\n\n    Returns:\n        bytes: JPEG diff image or None on failure\n    \"\"\"\n    ctx = multiprocessing.get_context('spawn')\n    parent_conn, child_conn = ctx.Pipe()\n\n    p = ctx.Process(\n        target=_worker_generate_diff,\n        args=(child_conn, img_bytes_from, img_bytes_to, threshold, blur_sigma, max_width, max_height)\n    )\n    p.start()\n\n    result = None\n    try:\n        # Wait for result (30 second timeout)\n        if parent_conn.poll(30):\n            result = parent_conn.recv()\n    except Exception as e:\n        print(f\"[Parent] Error receiving result: {e}\", flush=True)\n    finally:\n        # Always close pipe first\n        try:\n            parent_conn.close()\n        except:\n            pass\n\n        # Try graceful shutdown\n        p.join(timeout=5)\n        if p.is_alive():\n            print(\"[Parent] Process didn't exit gracefully, terminating\", flush=True)\n            p.terminate()\n            p.join(timeout=3)\n\n        # Force kill if still alive\n        if p.is_alive():\n            print(\"[Parent] Process didn't terminate, killing\", flush=True)\n            p.kill()\n            p.join(timeout=1)\n\n    return result\n\n\ndef calculate_change_percentage_isolated(img_bytes_from, img_bytes_to, threshold, blur_sigma, max_width, max_height):\n    \"\"\"\n    Calculate change percentage in isolated subprocess using handler.\n\n    Returns:\n        float: Change percentage\n    \"\"\"\n    ctx = multiprocessing.get_context('spawn')\n    parent_conn, child_conn = ctx.Pipe()\n\n    def _worker_calculate(conn):\n        try:\n            # Import handler inside worker\n            from .libvips_handler import LibvipsImageDiffHandler\n\n            handler = LibvipsImageDiffHandler()\n\n            # Load images\n            img_from = handler.load_from_bytes(img_bytes_from)\n            img_to = handler.load_from_bytes(img_bytes_to)\n\n            # Ensure same size\n            w1, h1 = handler.get_dimensions(img_from)\n            w2, h2 = handler.get_dimensions(img_to)\n            if (w1, h1) != (w2, h2):\n                img_from = handler.resize(img_from, w2, h2)\n\n            # Downscale\n            img_from = handler.resize(img_from, max_width, max_height)\n            img_to = handler.resize(img_to, max_width, max_height)\n\n            # Convert to grayscale\n            gray_from = handler.to_grayscale(img_from)\n            gray_to = handler.to_grayscale(img_to)\n\n            # Optional blur\n            gray_from = handler.gaussian_blur(gray_from, blur_sigma)\n            gray_to = handler.gaussian_blur(gray_to, blur_sigma)\n\n            # Calculate difference\n            diff = handler.absolute_difference(gray_from, gray_to)\n\n            # Threshold and get percentage\n            change_percentage, _ = handler.threshold(diff, int(threshold))\n\n            conn.send(float(change_percentage))\n\n        except Exception as e:\n            print(f\"[Worker] Calculate error: {e}\", flush=True)\n            conn.send(0.0)\n        finally:\n            conn.close()\n\n    p = ctx.Process(target=_worker_calculate, args=(child_conn,))\n    p.start()\n\n    result = 0.0\n    try:\n        if parent_conn.poll(30):\n            result = parent_conn.recv()\n    except Exception as e:\n        print(f\"[Parent] Calculate error receiving result: {e}\", flush=True)\n    finally:\n        # Always close pipe first\n        try:\n            parent_conn.close()\n        except:\n            pass\n\n        # Try graceful shutdown\n        p.join(timeout=5)\n        if p.is_alive():\n            print(\"[Parent] Calculate process didn't exit gracefully, terminating\", flush=True)\n            p.terminate()\n            p.join(timeout=3)\n\n        # Force kill if still alive\n        if p.is_alive():\n            print(\"[Parent] Calculate process didn't terminate, killing\", flush=True)\n            p.kill()\n            p.join(timeout=1)\n\n    return result\n\n\ndef compare_images_isolated(img_bytes_from, img_bytes_to, threshold, blur_sigma, min_change_percentage, crop_region=None):\n    \"\"\"\n    Compare images in isolated subprocess for change detection.\n\n    Args:\n        img_bytes_from: Previous screenshot bytes\n        img_bytes_to: Current screenshot bytes\n        threshold: Pixel difference threshold\n        blur_sigma: Gaussian blur sigma\n        min_change_percentage: Minimum percentage to trigger change detection\n        crop_region: Optional tuple (left, top, right, bottom) for cropping both images\n\n    Returns:\n        tuple: (changed_detected, change_percentage)\n    \"\"\"\n    print(f\"[Parent] Starting compare_images_isolated subprocess\", flush=True)\n    ctx = multiprocessing.get_context('spawn')\n    parent_conn, child_conn = ctx.Pipe()\n\n    def _worker_compare(conn):\n        try:\n            print(f\"[Worker] Compare worker starting\", flush=True)\n            # Import handler inside worker\n            from .libvips_handler import LibvipsImageDiffHandler\n\n            print(f\"[Worker] Initializing handler\", flush=True)\n            handler = LibvipsImageDiffHandler()\n\n            # Load images\n            print(f\"[Worker] Loading images (from={len(img_bytes_from)} bytes, to={len(img_bytes_to)} bytes)\", flush=True)\n            img_from = handler.load_from_bytes(img_bytes_from)\n            img_to = handler.load_from_bytes(img_bytes_to)\n            print(f\"[Worker] Images loaded\", flush=True)\n\n            # Crop if region specified\n            if crop_region:\n                print(f\"[Worker] Cropping to region {crop_region}\", flush=True)\n                left, top, right, bottom = crop_region\n                img_from = handler.crop(img_from, left, top, right, bottom)\n                img_to = handler.crop(img_to, left, top, right, bottom)\n                print(f\"[Worker] Cropping completed\", flush=True)\n\n            # Ensure same size\n            w1, h1 = handler.get_dimensions(img_from)\n            w2, h2 = handler.get_dimensions(img_to)\n            print(f\"[Worker] Image dimensions: from={w1}x{h1}, to={w2}x{h2}\", flush=True)\n            if (w1, h1) != (w2, h2):\n                print(f\"[Worker] Resizing to match dimensions\", flush=True)\n                img_from = handler.resize(img_from, w2, h2)\n\n            # Convert to grayscale\n            print(f\"[Worker] Converting to grayscale\", flush=True)\n            gray_from = handler.to_grayscale(img_from)\n            gray_to = handler.to_grayscale(img_to)\n\n            # Optional blur\n            # NOTE: gaussblur can hang in forked subprocesses due to LibVIPS threading\n            # Skip blur as a workaround - sigma=0.8 is subtle and comparison works without it\n            if blur_sigma > 0:\n                print(f\"[Worker] Skipping blur (sigma={blur_sigma}) due to LibVIPS threading issues in fork\", flush=True)\n                # gray_from = handler.gaussian_blur(gray_from, blur_sigma)\n                # gray_to = handler.gaussian_blur(gray_to, blur_sigma)\n\n            # Calculate difference\n            print(f\"[Worker] Calculating difference\", flush=True)\n            diff = handler.absolute_difference(gray_from, gray_to)\n\n            # Threshold and get percentage\n            print(f\"[Worker] Applying threshold ({threshold})\", flush=True)\n            change_percentage, _ = handler.threshold(diff, int(threshold))\n\n            # Determine if change detected\n            changed_detected = change_percentage > min_change_percentage\n\n            print(f\"[Worker] Comparison complete: changed={changed_detected}, percentage={change_percentage:.2f}%\", flush=True)\n            conn.send((changed_detected, float(change_percentage)))\n\n        except Exception as e:\n            print(f\"[Worker] Compare error: {e}\", flush=True)\n            import traceback\n            traceback.print_exc()\n            conn.send((False, 0.0))\n        finally:\n            conn.close()\n\n    p = ctx.Process(target=_worker_compare, args=(child_conn,))\n    print(f\"[Parent] Starting subprocess (pid will be assigned)\", flush=True)\n    p.start()\n    print(f\"[Parent] Subprocess started (pid={p.pid}), waiting for result (30s timeout)\", flush=True)\n\n    result = (False, 0.0)\n    try:\n        if parent_conn.poll(30):\n            print(f\"[Parent] Result available, receiving\", flush=True)\n            result = parent_conn.recv()\n            print(f\"[Parent] Result received: {result}\", flush=True)\n        else:\n            print(f\"[Parent] Timeout waiting for result after 30s\", flush=True)\n    except Exception as e:\n        print(f\"[Parent] Compare error receiving result: {e}\", flush=True)\n    finally:\n        # Always close pipe first\n        try:\n            parent_conn.close()\n        except:\n            pass\n\n        # Try graceful shutdown\n        import time\n        print(f\"[Parent] Waiting for subprocess to exit (5s timeout)\", flush=True)\n        join_start = time.time()\n        p.join(timeout=5)\n        join_elapsed = time.time() - join_start\n        print(f\"[Parent] First join took {join_elapsed:.2f}s\", flush=True)\n\n        if p.is_alive():\n            print(\"[Parent] Compare process didn't exit gracefully, terminating\", flush=True)\n            term_start = time.time()\n            p.terminate()\n            p.join(timeout=3)\n            term_elapsed = time.time() - term_start\n            print(f\"[Parent] Terminate+join took {term_elapsed:.2f}s\", flush=True)\n\n        # Force kill if still alive\n        if p.is_alive():\n            print(\"[Parent] Compare process didn't terminate, killing\", flush=True)\n            kill_start = time.time()\n            p.kill()\n            p.join(timeout=1)\n            kill_elapsed = time.time() - kill_start\n            print(f\"[Parent] Kill+join took {kill_elapsed:.2f}s\", flush=True)\n\n        print(f\"[Parent] Subprocess cleanup complete, returning result\", flush=True)\n\n    return result\n"
  },
  {
    "path": "changedetectionio/processors/image_ssim_diff/image_handler/isolated_opencv.py",
    "content": "\"\"\"\nOpenCV-based subprocess isolation for image comparison.\n\nOpenCV is much more stable in multiprocessing contexts than LibVIPS.\nNo threading issues, no fork problems, picklable functions.\n\"\"\"\n\nimport multiprocessing\nimport numpy as np\nfrom .. import POLL_TIMEOUT_ABSOLUTE\n\n# Public implementation name for logging\nIMPLEMENTATION_NAME = \"OpenCV\"\n\n\ndef _worker_compare(conn, img_bytes_from, img_bytes_to, pixel_difference_threshold, blur_sigma, crop_region):\n    \"\"\"\n    Worker function for image comparison (must be top-level for pickling with spawn).\n\n    Args:\n        conn: Pipe connection for sending results\n        img_bytes_from: Previous screenshot bytes\n        img_bytes_to: Current screenshot bytes\n        pixel_difference_threshold: Pixel-level sensitivity (0-255) - how different must a pixel be to count as changed\n        blur_sigma: Gaussian blur sigma\n        crop_region: Optional (left, top, right, bottom) crop coordinates\n    \"\"\"\n    import time\n    try:\n        import cv2\n\n        # CRITICAL: Disable OpenCV threading to prevent thread explosion\n        # With multiprocessing, each subprocess would otherwise spawn threads equal to CPU cores\n        # This causes excessive thread counts and memory overhead\n        # Research: https://medium.com/@rachittayal7/a-note-on-opencv-threads-performance-in-prod-d10180716fba\n        cv2.setNumThreads(1)\n\n        print(f\"[{time.time():.3f}] [Worker] Compare worker starting (threads=1 for memory optimization)\", flush=True)\n\n        # Decode images from bytes\n        print(f\"[{time.time():.3f}] [Worker] Loading images (from={len(img_bytes_from)} bytes, to={len(img_bytes_to)} bytes)\", flush=True)\n        img_from = cv2.imdecode(np.frombuffer(img_bytes_from, np.uint8), cv2.IMREAD_COLOR)\n        img_to = cv2.imdecode(np.frombuffer(img_bytes_to, np.uint8), cv2.IMREAD_COLOR)\n\n        # Check if decoding succeeded\n        if img_from is None:\n            raise ValueError(\"Failed to decode 'from' image - may be corrupt or unsupported format\")\n        if img_to is None:\n            raise ValueError(\"Failed to decode 'to' image - may be corrupt or unsupported format\")\n\n        print(f\"[{time.time():.3f}] [Worker] Images loaded: from={img_from.shape}, to={img_to.shape}\", flush=True)\n\n        # Crop if region specified\n        if crop_region:\n            print(f\"[{time.time():.3f}] [Worker] Cropping to region {crop_region}\", flush=True)\n            left, top, right, bottom = crop_region\n            img_from = img_from[top:bottom, left:right]\n            img_to = img_to[top:bottom, left:right]\n            print(f\"[{time.time():.3f}] [Worker] Cropped: from={img_from.shape}, to={img_to.shape}\", flush=True)\n\n        # Resize if dimensions don't match\n        if img_from.shape != img_to.shape:\n            print(f\"[{time.time():.3f}] [Worker] Resizing to match dimensions\", flush=True)\n            img_from = cv2.resize(img_from, (img_to.shape[1], img_to.shape[0]))\n\n        # Convert to grayscale\n        print(f\"[{time.time():.3f}] [Worker] Converting to grayscale\", flush=True)\n        gray_from = cv2.cvtColor(img_from, cv2.COLOR_BGR2GRAY)\n        gray_to = cv2.cvtColor(img_to, cv2.COLOR_BGR2GRAY)\n\n        # Optional Gaussian blur\n        if blur_sigma > 0:\n            print(f\"[{time.time():.3f}] [Worker] Applying Gaussian blur (sigma={blur_sigma})\", flush=True)\n            # OpenCV uses kernel size, convert sigma to kernel size: size = 2 * round(3*sigma) + 1\n            ksize = int(2 * round(3 * blur_sigma)) + 1\n            if ksize % 2 == 0:  # Must be odd\n                ksize += 1\n            gray_from = cv2.GaussianBlur(gray_from, (ksize, ksize), blur_sigma)\n            gray_to = cv2.GaussianBlur(gray_to, (ksize, ksize), blur_sigma)\n            print(f\"[{time.time():.3f}] [Worker] Blur applied (kernel={ksize}x{ksize})\", flush=True)\n\n        # Calculate absolute difference\n        print(f\"[{time.time():.3f}] [Worker] Calculating absolute difference\", flush=True)\n        diff = cv2.absdiff(gray_from, gray_to)\n\n        # Apply threshold\n        print(f\"[{time.time():.3f}] [Worker] Applying pixel difference threshold ({pixel_difference_threshold})\", flush=True)\n        _, thresholded = cv2.threshold(diff, int(pixel_difference_threshold), 255, cv2.THRESH_BINARY)\n\n        # Calculate change percentage\n        total_pixels = thresholded.size\n        changed_pixels = np.count_nonzero(thresholded)\n        change_percentage = (changed_pixels / total_pixels) * 100.0\n\n        print(f\"[{time.time():.3f}] [Worker] Comparison complete: percentage={change_percentage:.2f}%\", flush=True)\n        # Return only the score - let the caller decide if it's a \"change\"\n        conn.send(float(change_percentage))\n\n    except Exception as e:\n        print(f\"[{time.time():.3f}] [Worker] Error: {e}\", flush=True)\n        import traceback\n        traceback.print_exc()\n        # Send error info as dict so parent can re-raise\n        conn.send({'error': str(e), 'traceback': traceback.format_exc()})\n    finally:\n        conn.close()\n\n\nasync def compare_images_isolated(img_bytes_from, img_bytes_to, pixel_difference_threshold, blur_sigma, crop_region=None):\n    \"\"\"\n    Compare images in isolated subprocess using OpenCV (async-safe).\n\n    Args:\n        img_bytes_from: Previous screenshot bytes\n        img_bytes_to: Current screenshot bytes\n        pixel_difference_threshold: Pixel-level sensitivity (0-255) - how different must a pixel be to count as changed\n        blur_sigma: Gaussian blur sigma\n        crop_region: Optional (left, top, right, bottom) crop coordinates\n\n    Returns:\n        float: Change percentage (0-100)\n    \"\"\"\n    import time\n    import asyncio\n    print(f\"[{time.time():.3f}] [Parent] Starting OpenCV comparison subprocess\", flush=True)\n\n    # Use spawn method for clean process (no fork issues)\n    ctx = multiprocessing.get_context('spawn')\n    parent_conn, child_conn = ctx.Pipe()\n\n    p = ctx.Process(\n        target=_worker_compare,\n        args=(child_conn, img_bytes_from, img_bytes_to, pixel_difference_threshold, blur_sigma, crop_region)\n    )\n\n    print(f\"[{time.time():.3f}] [Parent] Starting subprocess\", flush=True)\n    p.start()\n    print(f\"[{time.time():.3f}] [Parent] Subprocess started (pid={p.pid}), waiting for result ({POLL_TIMEOUT_ABSOLUTE}s timeout)\", flush=True)\n\n    result = 0.0\n    try:\n        # Async-friendly polling: check in small intervals without blocking event loop\n        deadline = time.time() + POLL_TIMEOUT_ABSOLUTE\n        while time.time() < deadline:\n            # Run poll() in thread to avoid blocking event loop\n            has_data = await asyncio.to_thread(parent_conn.poll, 0.1)\n            if has_data:\n                print(f\"[{time.time():.3f}] [Parent] Result available, receiving\", flush=True)\n                result = await asyncio.to_thread(parent_conn.recv)\n                # Check if result is an error dict\n                if isinstance(result, dict) and 'error' in result:\n                    raise RuntimeError(f\"Image comparison failed: {result['error']}\")\n                print(f\"[{time.time():.3f}] [Parent] Result received: {result:.2f}%\", flush=True)\n                break\n            await asyncio.sleep(0)  # Yield control to event loop\n        else:\n            from loguru import logger\n            logger.critical(f\"[OpenCV subprocess] Timeout waiting for compare_images result after {POLL_TIMEOUT_ABSOLUTE}s (subprocess may be hung)\")\n            print(f\"[{time.time():.3f}] [Parent] Timeout waiting for result after {POLL_TIMEOUT_ABSOLUTE}s\", flush=True)\n            raise TimeoutError(f\"Image comparison subprocess timeout after {POLL_TIMEOUT_ABSOLUTE}s\")\n    except Exception as e:\n        print(f\"[{time.time():.3f}] [Parent] Error receiving result: {e}\", flush=True)\n        raise\n    finally:\n        # Always close pipe first\n        try:\n            parent_conn.close()\n        except:\n            pass\n\n        # Try graceful shutdown (async-safe)\n        print(f\"[{time.time():.3f}] [Parent] Waiting for subprocess to exit (5s timeout)\", flush=True)\n        join_start = time.time()\n        await asyncio.to_thread(p.join, 5)\n        join_elapsed = time.time() - join_start\n        print(f\"[{time.time():.3f}] [Parent] First join took {join_elapsed:.2f}s\", flush=True)\n\n        if p.is_alive():\n            print(f\"[{time.time():.3f}] [Parent] Process didn't exit gracefully, terminating\", flush=True)\n            term_start = time.time()\n            p.terminate()\n            await asyncio.to_thread(p.join, 3)\n            term_elapsed = time.time() - term_start\n            print(f\"[{time.time():.3f}] [Parent] Terminate+join took {term_elapsed:.2f}s\", flush=True)\n\n        # Force kill if still alive\n        if p.is_alive():\n            print(f\"[{time.time():.3f}] [Parent] Process didn't terminate, killing\", flush=True)\n            kill_start = time.time()\n            p.kill()\n            await asyncio.to_thread(p.join, 1)\n            kill_elapsed = time.time() - kill_start\n            print(f\"[{time.time():.3f}] [Parent] Kill+join took {kill_elapsed:.2f}s\", flush=True)\n\n        print(f\"[{time.time():.3f}] [Parent] Subprocess cleanup complete, returning result\", flush=True)\n\n    return result\n\n\ndef _worker_generate_diff(conn, img_bytes_from, img_bytes_to, pixel_difference_threshold, blur_sigma, max_width, max_height):\n    \"\"\"\n    Worker function for generating visual diff with red overlay.\n    \"\"\"\n    import time\n    try:\n        import cv2\n\n        cv2.setNumThreads(1)\n        print(f\"[{time.time():.3f}] [Worker] Generate diff worker starting\", flush=True)\n\n        # Decode images\n        img_from = cv2.imdecode(np.frombuffer(img_bytes_from, np.uint8), cv2.IMREAD_COLOR)\n        img_to = cv2.imdecode(np.frombuffer(img_bytes_to, np.uint8), cv2.IMREAD_COLOR)\n\n        # Resize if needed to match dimensions\n        if img_from.shape != img_to.shape:\n            img_from = cv2.resize(img_from, (img_to.shape[1], img_to.shape[0]))\n\n        # Downscale to max dimensions for faster processing\n        h, w = img_to.shape[:2]\n        if w > max_width or h > max_height:\n            scale = min(max_width / w, max_height / h)\n            new_w = int(w * scale)\n            new_h = int(h * scale)\n            img_from = cv2.resize(img_from, (new_w, new_h))\n            img_to = cv2.resize(img_to, (new_w, new_h))\n\n        # Convert to grayscale\n        gray_from = cv2.cvtColor(img_from, cv2.COLOR_BGR2GRAY)\n        gray_to = cv2.cvtColor(img_to, cv2.COLOR_BGR2GRAY)\n\n        # Optional blur\n        if blur_sigma > 0:\n            ksize = int(2 * round(3 * blur_sigma)) + 1\n            if ksize % 2 == 0:\n                ksize += 1\n            gray_from = cv2.GaussianBlur(gray_from, (ksize, ksize), blur_sigma)\n            gray_to = cv2.GaussianBlur(gray_to, (ksize, ksize), blur_sigma)\n\n        # Calculate difference\n        diff = cv2.absdiff(gray_from, gray_to)\n\n        # Apply threshold to get mask\n        _, mask = cv2.threshold(diff, int(pixel_difference_threshold), 255, cv2.THRESH_BINARY)\n\n        # Create red overlay on original 'to' image\n        # Where mask is 255 (changed), blend 50% red\n        overlay = img_to.copy()\n        overlay[:, :, 2] = np.where(mask > 0,\n                                     np.clip(overlay[:, :, 2] * 0.5 + 127, 0, 255).astype(np.uint8),\n                                     overlay[:, :, 2])\n        overlay[:, :, 0:2] = np.where(mask[:, :, np.newaxis] > 0,\n                                       (overlay[:, :, 0:2] * 0.5).astype(np.uint8),\n                                       overlay[:, :, 0:2])\n\n        # Encode as JPEG\n        _, encoded = cv2.imencode('.jpg', overlay, [cv2.IMWRITE_JPEG_QUALITY, 85])\n        diff_bytes = encoded.tobytes()\n\n        print(f\"[{time.time():.3f}] [Worker] Generated diff ({len(diff_bytes)} bytes)\", flush=True)\n        conn.send(diff_bytes)\n\n    except Exception as e:\n        print(f\"[{time.time():.3f}] [Worker] Generate diff error: {e}\", flush=True)\n        import traceback\n        traceback.print_exc()\n        # Send error info as dict so parent can re-raise\n        conn.send({'error': str(e), 'traceback': traceback.format_exc()})\n    finally:\n        conn.close()\n\n\nasync def generate_diff_isolated(img_bytes_from, img_bytes_to, pixel_difference_threshold, blur_sigma, max_width, max_height):\n    \"\"\"\n    Generate visual diff with red overlay in isolated subprocess (async-safe).\n\n    Returns:\n        bytes: JPEG diff image or None on failure\n    \"\"\"\n    import time\n    import asyncio\n    print(f\"[{time.time():.3f}] [Parent] Starting generate_diff subprocess\", flush=True)\n\n    ctx = multiprocessing.get_context('spawn')\n    parent_conn, child_conn = ctx.Pipe()\n\n    p = ctx.Process(\n        target=_worker_generate_diff,\n        args=(child_conn, img_bytes_from, img_bytes_to, pixel_difference_threshold, blur_sigma, max_width, max_height)\n    )\n\n    print(f\"[{time.time():.3f}] [Parent] Starting subprocess\", flush=True)\n    p.start()\n    print(f\"[{time.time():.3f}] [Parent] Subprocess started (pid={p.pid}), waiting for result ({POLL_TIMEOUT_ABSOLUTE}s timeout)\", flush=True)\n\n    result = None\n    try:\n        # Async-friendly polling: check in small intervals without blocking event loop\n        deadline = time.time() + POLL_TIMEOUT_ABSOLUTE\n        while time.time() < deadline:\n            # Run poll() in thread to avoid blocking event loop\n            has_data = await asyncio.to_thread(parent_conn.poll, 0.1)\n            if has_data:\n                print(f\"[{time.time():.3f}] [Parent] Result available, receiving\", flush=True)\n                result = await asyncio.to_thread(parent_conn.recv)\n                # Check if result is an error dict\n                if isinstance(result, dict) and 'error' in result:\n                    raise RuntimeError(f\"Generate diff failed: {result['error']}\")\n                print(f\"[{time.time():.3f}] [Parent] Result received ({len(result) if result else 0} bytes)\", flush=True)\n                break\n            await asyncio.sleep(0)  # Yield control to event loop\n        else:\n            from loguru import logger\n            logger.critical(f\"[OpenCV subprocess] Timeout waiting for generate_diff result after {POLL_TIMEOUT_ABSOLUTE}s (subprocess may be hung)\")\n            print(f\"[{time.time():.3f}] [Parent] Timeout waiting for result after {POLL_TIMEOUT_ABSOLUTE}s\", flush=True)\n            raise TimeoutError(f\"Generate diff subprocess timeout after {POLL_TIMEOUT_ABSOLUTE}s\")\n    except Exception as e:\n        print(f\"[{time.time():.3f}] [Parent] Error receiving diff: {e}\", flush=True)\n        raise\n    finally:\n        # Always close pipe first\n        try:\n            parent_conn.close()\n        except:\n            pass\n\n        # Try graceful shutdown (async-safe)\n        print(f\"[{time.time():.3f}] [Parent] Waiting for subprocess to exit (5s timeout)\", flush=True)\n        join_start = time.time()\n        await asyncio.to_thread(p.join, 5)\n        join_elapsed = time.time() - join_start\n        print(f\"[{time.time():.3f}] [Parent] First join took {join_elapsed:.2f}s\", flush=True)\n\n        if p.is_alive():\n            print(f\"[{time.time():.3f}] [Parent] Process didn't exit gracefully, terminating\", flush=True)\n            term_start = time.time()\n            p.terminate()\n            await asyncio.to_thread(p.join, 3)\n            term_elapsed = time.time() - term_start\n            print(f\"[{time.time():.3f}] [Parent] Terminate+join took {term_elapsed:.2f}s\", flush=True)\n\n        if p.is_alive():\n            print(f\"[{time.time():.3f}] [Parent] Process didn't terminate, killing\", flush=True)\n            kill_start = time.time()\n            p.kill()\n            await asyncio.to_thread(p.join, 1)\n            kill_elapsed = time.time() - kill_start\n            print(f\"[{time.time():.3f}] [Parent] Kill+join took {kill_elapsed:.2f}s\", flush=True)\n\n        print(f\"[{time.time():.3f}] [Parent] Subprocess cleanup complete, returning result\", flush=True)\n\n    return result\n\n\ndef _worker_draw_bounding_box(conn, img_bytes, x, y, width, height, color, thickness):\n    \"\"\"\n    Worker function for drawing bounding box on image.\n    \"\"\"\n    import time\n    try:\n        import cv2\n\n        cv2.setNumThreads(1)\n        print(f\"[{time.time():.3f}] [Worker] Draw bounding box worker starting\", flush=True)\n\n        # Decode image\n        img = cv2.imdecode(np.frombuffer(img_bytes, np.uint8), cv2.IMREAD_COLOR)\n        if img is None:\n            print(f\"[{time.time():.3f}] [Worker] Failed to decode image\", flush=True)\n            conn.send(None)\n            return\n\n        # Draw rectangle (BGR format)\n        cv2.rectangle(img, (x, y), (x + width, y + height), color, thickness)\n\n        # Encode back to PNG\n        _, encoded = cv2.imencode('.png', img)\n        result_bytes = encoded.tobytes()\n\n        print(f\"[{time.time():.3f}] [Worker] Bounding box drawn ({len(result_bytes)} bytes)\", flush=True)\n        conn.send(result_bytes)\n\n    except Exception as e:\n        print(f\"[{time.time():.3f}] [Worker] Draw bounding box error: {e}\", flush=True)\n        import traceback\n        traceback.print_exc()\n        # Send error info as dict so parent can re-raise\n        conn.send({'error': str(e), 'traceback': traceback.format_exc()})\n    finally:\n        conn.close()\n\n\nasync def draw_bounding_box_isolated(img_bytes, x, y, width, height, color=(255, 0, 0), thickness=3):\n    \"\"\"\n    Draw bounding box on image in isolated subprocess (async-safe).\n\n    Args:\n        img_bytes: Image data as bytes\n        x: Left coordinate\n        y: Top coordinate\n        width: Box width\n        height: Box height\n        color: BGR color tuple (default: blue)\n        thickness: Line thickness in pixels\n\n    Returns:\n        bytes: PNG image with bounding box or None on failure\n    \"\"\"\n    import time\n    import asyncio\n    print(f\"[{time.time():.3f}] [Parent] Starting draw_bounding_box subprocess\", flush=True)\n\n    ctx = multiprocessing.get_context('spawn')\n    parent_conn, child_conn = ctx.Pipe()\n\n    p = ctx.Process(\n        target=_worker_draw_bounding_box,\n        args=(child_conn, img_bytes, x, y, width, height, color, thickness)\n    )\n\n    print(f\"[{time.time():.3f}] [Parent] Starting subprocess\", flush=True)\n    p.start()\n    print(f\"[{time.time():.3f}] [Parent] Subprocess started (pid={p.pid}), waiting for result ({POLL_TIMEOUT_ABSOLUTE}s timeout)\", flush=True)\n\n    result = None\n    try:\n        # Async-friendly polling: check in small intervals without blocking event loop\n        deadline = time.time() + POLL_TIMEOUT_ABSOLUTE\n        while time.time() < deadline:\n            # Run poll() in thread to avoid blocking event loop\n            has_data = await asyncio.to_thread(parent_conn.poll, 0.1)\n            if has_data:\n                print(f\"[{time.time():.3f}] [Parent] Result available, receiving\", flush=True)\n                # Run recv() in thread too\n                result = await asyncio.to_thread(parent_conn.recv)\n                # Check if result is an error dict\n                if isinstance(result, dict) and 'error' in result:\n                    raise RuntimeError(f\"Draw bounding box failed: {result['error']}\")\n                print(f\"[{time.time():.3f}] [Parent] Result received ({len(result) if result else 0} bytes)\", flush=True)\n                break\n            # Yield control to event loop\n            await asyncio.sleep(0)\n        else:\n            from loguru import logger\n            logger.critical(f\"[OpenCV subprocess] Timeout waiting for draw_bounding_box result after {POLL_TIMEOUT_ABSOLUTE}s (subprocess may be hung)\")\n            print(f\"[{time.time():.3f}] [Parent] Timeout waiting for result after {POLL_TIMEOUT_ABSOLUTE}s\", flush=True)\n            raise TimeoutError(f\"Draw bounding box subprocess timeout after {POLL_TIMEOUT_ABSOLUTE}s\")\n    except Exception as e:\n        print(f\"[{time.time():.3f}] [Parent] Error receiving result: {e}\", flush=True)\n        raise\n    finally:\n        # Always close pipe first\n        try:\n            parent_conn.close()\n        except:\n            pass\n\n        # Try graceful shutdown (run join in thread to avoid blocking)\n        print(f\"[{time.time():.3f}] [Parent] Waiting for subprocess to exit (3s timeout)\", flush=True)\n        join_start = time.time()\n        await asyncio.to_thread(p.join, 3)\n        join_elapsed = time.time() - join_start\n        print(f\"[{time.time():.3f}] [Parent] First join took {join_elapsed:.2f}s\", flush=True)\n\n        if p.is_alive():\n            print(f\"[{time.time():.3f}] [Parent] Process didn't exit gracefully, terminating\", flush=True)\n            term_start = time.time()\n            p.terminate()\n            await asyncio.to_thread(p.join, 2)\n            term_elapsed = time.time() - term_start\n            print(f\"[{time.time():.3f}] [Parent] Terminate+join took {term_elapsed:.2f}s\", flush=True)\n\n        if p.is_alive():\n            print(f\"[{time.time():.3f}] [Parent] Process didn't terminate, killing\", flush=True)\n            kill_start = time.time()\n            p.kill()\n            await asyncio.to_thread(p.join, 1)\n            kill_elapsed = time.time() - kill_start\n            print(f\"[{time.time():.3f}] [Parent] Kill+join took {kill_elapsed:.2f}s\", flush=True)\n\n        print(f\"[{time.time():.3f}] [Parent] Subprocess cleanup complete, returning result\", flush=True)\n\n    return result\n\n\ndef _worker_calculate_percentage(conn, img_bytes_from, img_bytes_to, pixel_difference_threshold, blur_sigma, max_width, max_height):\n    \"\"\"\n    Worker function for calculating change percentage.\n    \"\"\"\n    import time\n    try:\n        import cv2\n\n        cv2.setNumThreads(1)\n\n        # Decode images\n        img_from = cv2.imdecode(np.frombuffer(img_bytes_from, np.uint8), cv2.IMREAD_COLOR)\n        img_to = cv2.imdecode(np.frombuffer(img_bytes_to, np.uint8), cv2.IMREAD_COLOR)\n\n        # Resize if needed\n        if img_from.shape != img_to.shape:\n            img_from = cv2.resize(img_from, (img_to.shape[1], img_to.shape[0]))\n\n        # Downscale to max dimensions\n        h, w = img_to.shape[:2]\n        if w > max_width or h > max_height:\n            scale = min(max_width / w, max_height / h)\n            new_w = int(w * scale)\n            new_h = int(h * scale)\n            img_from = cv2.resize(img_from, (new_w, new_h))\n            img_to = cv2.resize(img_to, (new_w, new_h))\n\n        # Convert to grayscale\n        gray_from = cv2.cvtColor(img_from, cv2.COLOR_BGR2GRAY)\n        gray_to = cv2.cvtColor(img_to, cv2.COLOR_BGR2GRAY)\n\n        # Optional blur\n        if blur_sigma > 0:\n            ksize = int(2 * round(3 * blur_sigma)) + 1\n            if ksize % 2 == 0:\n                ksize += 1\n            gray_from = cv2.GaussianBlur(gray_from, (ksize, ksize), blur_sigma)\n            gray_to = cv2.GaussianBlur(gray_to, (ksize, ksize), blur_sigma)\n\n        # Calculate difference\n        diff = cv2.absdiff(gray_from, gray_to)\n\n        # Apply threshold\n        _, thresholded = cv2.threshold(diff, int(pixel_difference_threshold), 255, cv2.THRESH_BINARY)\n\n        # Calculate percentage\n        total_pixels = thresholded.size\n        changed_pixels = np.count_nonzero(thresholded)\n        change_percentage = (changed_pixels / total_pixels) * 100.0\n\n        conn.send(float(change_percentage))\n\n    except Exception as e:\n        print(f\"[{time.time():.3f}] [Worker] Calculate percentage error: {e}\", flush=True)\n        import traceback\n        traceback.print_exc()\n        # Send error info as dict so parent can re-raise\n        conn.send({'error': str(e), 'traceback': traceback.format_exc()})\n    finally:\n        conn.close()\n\n\nasync def calculate_change_percentage_isolated(img_bytes_from, img_bytes_to, pixel_difference_threshold, blur_sigma, max_width, max_height):\n    \"\"\"\n    Calculate change percentage in isolated subprocess (async-safe).\n\n    Returns:\n        float: Change percentage\n    \"\"\"\n    import time\n    import asyncio\n    print(f\"[{time.time():.3f}] [Parent] Starting calculate_percentage subprocess\", flush=True)\n\n    ctx = multiprocessing.get_context('spawn')\n    parent_conn, child_conn = ctx.Pipe()\n\n    p = ctx.Process(\n        target=_worker_calculate_percentage,\n        args=(child_conn, img_bytes_from, img_bytes_to, pixel_difference_threshold, blur_sigma, max_width, max_height)\n    )\n\n    print(f\"[{time.time():.3f}] [Parent] Starting subprocess\", flush=True)\n    p.start()\n    print(f\"[{time.time():.3f}] [Parent] Subprocess started (pid={p.pid}), waiting for result ({POLL_TIMEOUT_ABSOLUTE}s timeout)\", flush=True)\n\n    result = 0.0\n    try:\n        # Async-friendly polling: check in small intervals without blocking event loop\n        deadline = time.time() + POLL_TIMEOUT_ABSOLUTE\n        while time.time() < deadline:\n            # Run poll() in thread to avoid blocking event loop\n            has_data = await asyncio.to_thread(parent_conn.poll, 0.1)\n            if has_data:\n                print(f\"[{time.time():.3f}] [Parent] Result available, receiving\", flush=True)\n                result = await asyncio.to_thread(parent_conn.recv)\n                # Check if result is an error dict\n                if isinstance(result, dict) and 'error' in result:\n                    raise RuntimeError(f\"Calculate change percentage failed: {result['error']}\")\n                print(f\"[{time.time():.3f}] [Parent] Result received: {result:.2f}%\", flush=True)\n                break\n            await asyncio.sleep(0)  # Yield control to event loop\n        else:\n            from loguru import logger\n            logger.critical(f\"[OpenCV subprocess] Timeout waiting for calculate_change_percentage result after {POLL_TIMEOUT_ABSOLUTE}s (subprocess may be hung)\")\n            print(f\"[{time.time():.3f}] [Parent] Timeout waiting for result after {POLL_TIMEOUT_ABSOLUTE}s\", flush=True)\n            raise TimeoutError(f\"Calculate change percentage subprocess timeout after {POLL_TIMEOUT_ABSOLUTE}s\")\n    except Exception as e:\n        print(f\"[{time.time():.3f}] [Parent] Error receiving percentage: {e}\", flush=True)\n        raise\n    finally:\n        # Always close pipe first\n        try:\n            parent_conn.close()\n        except:\n            pass\n\n        # Try graceful shutdown (async-safe)\n        print(f\"[{time.time():.3f}] [Parent] Waiting for subprocess to exit (5s timeout)\", flush=True)\n        join_start = time.time()\n        await asyncio.to_thread(p.join, 5)\n        join_elapsed = time.time() - join_start\n        print(f\"[{time.time():.3f}] [Parent] First join took {join_elapsed:.2f}s\", flush=True)\n\n        if p.is_alive():\n            print(f\"[{time.time():.3f}] [Parent] Process didn't exit gracefully, terminating\", flush=True)\n            term_start = time.time()\n            p.terminate()\n            await asyncio.to_thread(p.join, 3)\n            term_elapsed = time.time() - term_start\n            print(f\"[{time.time():.3f}] [Parent] Terminate+join took {term_elapsed:.2f}s\", flush=True)\n\n        if p.is_alive():\n            print(f\"[{time.time():.3f}] [Parent] Process didn't terminate, killing\", flush=True)\n            kill_start = time.time()\n            p.kill()\n            await asyncio.to_thread(p.join, 1)\n            kill_elapsed = time.time() - kill_start\n            print(f\"[{time.time():.3f}] [Parent] Kill+join took {kill_elapsed:.2f}s\", flush=True)\n\n        print(f\"[{time.time():.3f}] [Parent] Subprocess cleanup complete, returning result\", flush=True)\n\n    return result\n"
  },
  {
    "path": "changedetectionio/processors/image_ssim_diff/image_handler/libvips_handler.py",
    "content": "\"\"\"\nLibVIPS implementation of ImageDiffHandler.\n\nUses pyvips for high-performance image processing with streaming architecture\nand low memory footprint. Ideal for large screenshots (8000px+).\n\"\"\"\n\nfrom __future__ import annotations\nimport os\nfrom typing import Tuple, Any, TYPE_CHECKING\nfrom loguru import logger\n\nif TYPE_CHECKING:\n    import pyvips\n\ntry:\n    import pyvips\n    PYVIPS_AVAILABLE = True\nexcept ImportError:\n    PYVIPS_AVAILABLE = False\n    logger.warning(\"pyvips not available - install with: pip install pyvips\")\n\nfrom . import ImageDiffHandler\n\n\nclass LibvipsImageDiffHandler(ImageDiffHandler):\n    \"\"\"\n    LibVIPS implementation using streaming architecture.\n\n    Benefits:\n    - 3x faster than ImageMagick\n    - 5x less memory than PIL\n    - Automatic multi-threading\n    - Streaming - processes images in chunks\n    \"\"\"\n\n    def __init__(self):\n        if not PYVIPS_AVAILABLE:\n            raise ImportError(\"pyvips is not installed. Install with: pip install pyvips\")\n\n    def load_from_bytes(self, img_bytes: bytes) -> pyvips.Image:\n        \"\"\"Load image from bytes using libvips streaming.\"\"\"\n        return pyvips.Image.new_from_buffer(img_bytes, '')\n\n    def save_to_bytes(self, img: pyvips.Image, format: str = 'png', quality: int = 85) -> bytes:\n        \"\"\"\n        Save image to bytes using temp file.\n\n        Note: Uses temp file instead of write_to_buffer() to avoid C memory leak.\n        See: https://github.com/libvips/pyvips/issues/234\n        \"\"\"\n        import tempfile\n\n        format = format.lower()\n\n        try:\n            if format == 'png':\n                suffix = '.png'\n                write_args = {'compression': 6}\n            elif format in ['jpg', 'jpeg']:\n                suffix = '.jpg'\n                write_args = {'Q': quality}\n            else:\n                raise ValueError(f\"Unsupported format: {format}\")\n\n            # Use temp file to avoid write_to_buffer() memory leak\n            with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as tmp:\n                temp_path = tmp.name\n\n            # Write to file\n            img.write_to_file(temp_path, **write_args)\n\n            # Read bytes and clean up\n            with open(temp_path, 'rb') as f:\n                image_bytes = f.read()\n\n            os.unlink(temp_path)\n            return image_bytes\n\n        except Exception as e:\n            logger.error(f\"Failed to save via temp file: {e}\")\n            # Fallback to write_to_buffer if temp file fails\n            if format == 'png':\n                return img.write_to_buffer('.png', compression=6)\n            else:\n                return img.write_to_buffer('.jpg', Q=quality)\n\n    def crop(self, img: pyvips.Image, left: int, top: int, right: int, bottom: int) -> pyvips.Image:\n        \"\"\"Crop image using libvips.\"\"\"\n        width = right - left\n        height = bottom - top\n        return img.crop(left, top, width, height)\n\n    def resize(self, img: pyvips.Image, max_width: int, max_height: int) -> pyvips.Image:\n        \"\"\"\n        Resize image maintaining aspect ratio.\n\n        Uses thumbnail_image for efficient downscaling with streaming.\n        \"\"\"\n        width, height = img.width, img.height\n\n        if width <= max_width and height <= max_height:\n            return img\n\n        # Calculate scaling to fit within max dimensions\n        width_ratio = max_width / width if width > max_width else 1.0\n        height_ratio = max_height / height if height > max_height else 1.0\n        ratio = min(width_ratio, height_ratio)\n\n        new_width = int(width * ratio)\n        new_height = int(height * ratio)\n\n        logger.debug(f\"Resizing image: {width}x{height} -> {new_width}x{new_height}\")\n\n        # thumbnail_image is faster than resize for downscaling\n        return img.thumbnail_image(new_width, height=new_height)\n\n    def get_dimensions(self, img: pyvips.Image) -> Tuple[int, int]:\n        \"\"\"Get image dimensions.\"\"\"\n        return (img.width, img.height)\n\n    def to_grayscale(self, img: pyvips.Image) -> pyvips.Image:\n        \"\"\"Convert to grayscale using 'b-w' colorspace.\"\"\"\n        return img.colourspace('b-w')\n\n    def gaussian_blur(self, img: pyvips.Image, sigma: float) -> pyvips.Image:\n        \"\"\"Apply Gaussian blur.\"\"\"\n        if sigma > 0:\n            return img.gaussblur(sigma)\n        return img\n\n    def absolute_difference(self, img1: pyvips.Image, img2: pyvips.Image) -> pyvips.Image:\n        \"\"\"\n        Calculate absolute difference using operator overloading.\n\n        LibVIPS supports arithmetic operations between images.\n        \"\"\"\n        return (img1 - img2).abs()\n\n    def threshold(self, img: pyvips.Image, threshold_value: int) -> Tuple[float, pyvips.Image]:\n        \"\"\"\n        Apply threshold and calculate change percentage.\n\n        Uses ifthenelse for efficient thresholding.\n        \"\"\"\n        # Create binary mask: pixels above threshold = 255, others = 0\n        mask = (img > threshold_value).ifthenelse(255, 0)\n\n        # Calculate percentage by averaging mask values\n        # avg() returns mean pixel value (0-255)\n        # Divide by 255 to get proportion, multiply by 100 for percentage\n        mean_value = mask.avg()\n        change_percentage = (mean_value / 255.0) * 100.0\n\n        return float(change_percentage), mask\n\n    def apply_red_overlay(self, img: pyvips.Image, mask: pyvips.Image) -> bytes:\n        \"\"\"\n        Apply red overlay where mask is True (50% blend).\n\n        Args:\n            img: Color image (will be converted to RGB if needed)\n            mask: Binary mask (255 where changed, 0 elsewhere)\n\n        Returns:\n            JPEG bytes with red overlay\n        \"\"\"\n        import tempfile\n\n        # Ensure RGB colorspace\n        if img.bands == 1:\n            img = img.colourspace('srgb')\n\n        # Normalize mask to 0-1 range for blending\n        mask_normalized = mask / 255.0\n\n        # Split into R, G, B channels\n        channels = img.bandsplit()\n        r, g, b = channels[0], channels[1], channels[2]\n\n        # Apply red overlay (50% blend):\n        # Where mask is 1: blend 50% original with 50% red (255)\n        # Where mask is 0: keep original\n        r = r * (1 - mask_normalized * 0.5) + 127.5 * mask_normalized\n        g = g * (1 - mask_normalized * 0.5)\n        b = b * (1 - mask_normalized * 0.5)\n\n        # Recombine channels\n        result = r.bandjoin([g, b])\n\n        # CRITICAL: Use temp file instead of write_to_buffer()\n        # write_to_buffer() leaks C memory that isn't returned to OS\n        # See: https://github.com/libvips/pyvips/issues/234\n        try:\n            with tempfile.NamedTemporaryFile(suffix='.jpg', delete=False) as tmp:\n                temp_path = tmp.name\n\n            # Write to file (doesn't leak like write_to_buffer)\n            result.write_to_file(temp_path, Q=85)\n\n            # Read bytes and clean up\n            with open(temp_path, 'rb') as f:\n                image_bytes = f.read()\n\n            os.unlink(temp_path)\n            return image_bytes\n\n        except Exception as e:\n            logger.error(f\"Failed to write image via temp file: {e}\")\n            # Fallback to write_to_buffer if temp file fails\n            return result.write_to_buffer('.jpg', Q=85)\n\n    def close(self, img: pyvips.Image) -> None:\n        \"\"\"\n        LibVIPS uses automatic reference counting.\n\n        No explicit cleanup needed - memory freed when references drop to zero.\n        \"\"\"\n        pass\n\n    def find_template(\n        self,\n        img: pyvips.Image,\n        template_img: pyvips.Image,\n        original_bbox: Tuple[int, int, int, int],\n        search_tolerance: float = 0.2\n    ) -> Tuple[int, int, int, int]:\n        \"\"\"\n        Find template in image using OpenCV template matching.\n\n        Note: This temporarily converts to numpy for OpenCV operations since\n        libvips doesn't have template matching built-in.\n        \"\"\"\n        import cv2\n        import numpy as np\n\n        try:\n            left, top, right, bottom = original_bbox\n            width = right - left\n            height = bottom - top\n\n            # Calculate search region\n            margin_x = int(width * search_tolerance)\n            margin_y = int(height * search_tolerance)\n\n            search_left = max(0, left - margin_x)\n            search_top = max(0, top - margin_y)\n            search_right = min(img.width, right + margin_x)\n            search_bottom = min(img.height, bottom + margin_y)\n\n            # Crop search region\n            search_region = self.crop(img, search_left, search_top, search_right, search_bottom)\n\n            # Convert to numpy arrays for OpenCV\n            search_array = np.ndarray(\n                buffer=search_region.write_to_memory(),\n                dtype=np.uint8,\n                shape=[search_region.height, search_region.width, search_region.bands]\n            )\n            template_array = np.ndarray(\n                buffer=template_img.write_to_memory(),\n                dtype=np.uint8,\n                shape=[template_img.height, template_img.width, template_img.bands]\n            )\n\n            # Convert to grayscale\n            if len(search_array.shape) == 3:\n                search_gray = cv2.cvtColor(search_array, cv2.COLOR_RGB2GRAY)\n            else:\n                search_gray = search_array\n\n            if len(template_array.shape) == 3:\n                template_gray = cv2.cvtColor(template_array, cv2.COLOR_RGB2GRAY)\n            else:\n                template_gray = template_array\n\n            logger.debug(f\"Searching for template in region: ({search_left}, {search_top}) to ({search_right}, {search_bottom})\")\n\n            # Perform template matching\n            result = cv2.matchTemplate(search_gray, template_gray, cv2.TM_CCOEFF_NORMED)\n            min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(result)\n\n            logger.debug(f\"Template matching confidence: {max_val:.2%}\")\n\n            # Check if match is good enough (80% confidence threshold)\n            if max_val >= 0.8:\n                # Calculate new bounding box in original image coordinates\n                match_x = search_left + max_loc[0]\n                match_y = search_top + max_loc[1]\n\n                new_bbox = (match_x, match_y, match_x + width, match_y + height)\n\n                # Calculate movement distance\n                move_x = abs(match_x - left)\n                move_y = abs(match_y - top)\n\n                logger.info(f\"Template found at ({match_x}, {match_y}), \"\n                           f\"moved {move_x}px horizontally, {move_y}px vertically, \"\n                           f\"confidence: {max_val:.2%}\")\n\n                return new_bbox\n            else:\n                logger.warning(f\"Template match confidence too low: {max_val:.2%} (need 80%)\")\n                return None\n\n        except Exception as e:\n            logger.error(f\"Template matching error: {e}\")\n            return None\n\n    def save_template(\n        self,\n        img: pyvips.Image,\n        bbox: Tuple[int, int, int, int],\n        output_path: str\n    ) -> bool:\n        \"\"\"\n        Save a cropped region as a template file.\n        \"\"\"\n        import os\n\n        try:\n            left, top, right, bottom = bbox\n            width = right - left\n            height = bottom - top\n\n            # Ensure output directory exists\n            os.makedirs(os.path.dirname(output_path), exist_ok=True)\n\n            # Crop template region\n            template = self.crop(img, left, top, right, bottom)\n\n            # Save as PNG\n            template.write_to_file(output_path, compression=6)\n\n            logger.info(f\"Saved template: {output_path} ({width}x{height}px)\")\n            return True\n\n        except Exception as e:\n            logger.error(f\"Failed to save template: {e}\")\n            return False\n"
  },
  {
    "path": "changedetectionio/processors/image_ssim_diff/preview.py",
    "content": "\"\"\"\nPreview rendering for SSIM screenshot processor.\n\nRenders images properly in the browser instead of showing raw bytes.\n\"\"\"\n\nfrom flask_babel import gettext\nfrom loguru import logger\n\n\ndef get_asset(asset_name, watch, datastore, request):\n    \"\"\"\n    Get processor-specific binary assets for preview streaming.\n\n    This function supports serving images as separate HTTP responses instead\n    of embedding them as base64 in the HTML template, solving memory issues\n    with large screenshots.\n\n    Supported assets:\n    - 'screenshot': The screenshot for the specified version\n\n    Args:\n        asset_name: Name of the asset to retrieve ('screenshot')\n        watch: Watch object\n        datastore: Datastore object\n        request: Flask request (for version query param)\n\n    Returns:\n        tuple: (binary_data, content_type, cache_control_header) or None if not found\n    \"\"\"\n    if asset_name != 'screenshot':\n        return None\n\n    versions = list(watch.history.keys())\n    if len(versions) == 0:\n        return None\n\n    # Get the version from query string (default: latest)\n    preferred_version = request.args.get('version')\n    timestamp = versions[-1]\n    if preferred_version and preferred_version in versions:\n        timestamp = preferred_version\n\n    try:\n        screenshot_bytes = watch.get_history_snapshot(timestamp=timestamp)\n\n        # Verify we got bytes (should always be bytes for image files)\n        if not isinstance(screenshot_bytes, bytes):\n            logger.error(f\"Expected bytes but got {type(screenshot_bytes)} for screenshot at {timestamp}\")\n            return None\n\n        # Detect image format using puremagic (same as Watch.py)\n        try:\n            import puremagic\n            detections = puremagic.magic_string(screenshot_bytes[:2048])\n            if detections:\n                mime_type = detections[0].mime_type\n                logger.trace(f\"Detected MIME type: {mime_type}\")\n            else:\n                mime_type = 'image/png'  # Default fallback\n        except Exception as e:\n            logger.warning(f\"puremagic detection failed: {e}, using 'image/png' fallback\")\n            mime_type = 'image/png'\n\n        return (screenshot_bytes, mime_type, 'public, max-age=10')\n\n    except Exception as e:\n        logger.error(f\"Failed to load screenshot for preview asset: {e}\")\n        return None\n\n\ndef render(watch, datastore, request, url_for, render_template, flash, redirect):\n    \"\"\"\n    Render the preview page for screenshot watches.\n\n    Args:\n        watch: Watch object\n        datastore: Datastore object\n        request: Flask request\n        url_for: Flask url_for function\n        render_template: Flask render_template function\n        flash: Flask flash function\n        redirect: Flask redirect function\n\n    Returns:\n        Rendered template or redirect\n    \"\"\"\n    versions = list(watch.history.keys())\n\n    if len(versions) == 0:\n        flash(gettext(\"Preview unavailable - No snapshots captured yet\"), \"error\")\n        return redirect(url_for('watchlist.index'))\n\n    # Get the version to display (default: latest)\n    preferred_version = request.args.get('version')\n    timestamp = versions[-1]\n    if preferred_version and preferred_version in versions:\n        timestamp = preferred_version\n\n    # Render custom template for image preview\n    # Screenshot is now served via separate /processor-asset/ endpoint instead of base64\n    # This significantly reduces memory usage by not embedding large images in HTML\n    return render_template(\n        'image_ssim_diff/preview.html',\n        watch=watch,\n        uuid=watch.get('uuid'),\n        versions=versions,\n        timestamp=timestamp,\n        current_diff_url=watch['url']\n    )\n"
  },
  {
    "path": "changedetectionio/processors/image_ssim_diff/processor.py",
    "content": "\"\"\"\nCore fast screenshot comparison processor.\n\nUses OpenCV with subprocess isolation for high-performance, low-memory\nimage processing. All operations run in isolated subprocesses for complete\nmemory cleanup and stability.\n\"\"\"\n\nimport hashlib\nimport time\nfrom loguru import logger\nfrom changedetectionio.processors.exceptions import ProcessorException\nfrom . import SCREENSHOT_COMPARISON_THRESHOLD_OPTIONS_DEFAULT, PROCESSOR_CONFIG_NAME, OPENCV_BLUR_SIGMA\nfrom ..base import difference_detection_processor, SCREENSHOT_FORMAT_PNG\n\n# All image operations now use OpenCV via isolated_opencv subprocess handler\n# Template matching temporarily disabled pending OpenCV implementation\n\n# Translation marker for extraction\ndef _(x): return x\nname = _('Visual / Image screenshot change detection')\ndescription = _('Compares screenshots using fast OpenCV algorithm, 10-100x faster than SSIM')\ndel _\nprocessor_weight = 2\nlist_badge_text = \"Visual\"\n\nclass perform_site_check(difference_detection_processor):\n    \"\"\"Fast screenshot comparison processor using OpenCV.\"\"\"\n\n    # Override to use PNG format for better image comparison (JPEG compression creates noise)\n    screenshot_format = SCREENSHOT_FORMAT_PNG\n\n    def run_changedetection(self, watch, force_reprocess=False):\n        \"\"\"\n        Perform screenshot comparison using OpenCV subprocess handler.\n\n        Returns:\n            tuple: (changed_detected, update_obj, screenshot_bytes)\n        \"\"\"\n        now = time.time()\n        # Get the current screenshot\n        if not self.fetcher.screenshot:\n            raise ProcessorException(\n                message=\"No screenshot available. Ensure the watch is configured to use a real browser.\",\n                url=watch.get('url')\n            )\n        self.screenshot = self.fetcher.screenshot\n        self.xpath_data = self.fetcher.xpath_data\n\n        # Quick MD5 check - skip expensive comparison if images are identical\n        from changedetectionio.content_fetchers.exceptions import checksumFromPreviousCheckWasTheSame\n        current_md5 = hashlib.md5(self.screenshot).hexdigest()\n        previous_md5 = watch.get('previous_md5')\n        if previous_md5 and current_md5 == previous_md5:\n            logger.debug(f\"UUID: {watch.get('uuid')} - Screenshot MD5 unchanged ({current_md5}), skipping comparison\")\n            raise checksumFromPreviousCheckWasTheSame()\n        else:\n            logger.debug(f\"UUID: {watch.get('uuid')} - Screenshot MD5 changed\")\n\n\n\n        # Check if bounding box is set (for drawn area mode)\n        # Read from processor-specific config JSON file (named after processor)\n        crop_region = None\n\n        processor_config = self.get_extra_watch_config(PROCESSOR_CONFIG_NAME)\n        bounding_box = processor_config.get('bounding_box') if processor_config else None\n\n\n        # Get pixel difference threshold sensitivity (per-watch > global)\n        # This controls how different a pixel must be (0-255 scale) to count as \"changed\"\n        pixel_difference_threshold_sensitivity = processor_config.get('pixel_difference_threshold_sensitivity')\n        if not pixel_difference_threshold_sensitivity:\n            pixel_difference_threshold_sensitivity = self.datastore.data['settings']['application'].get('pixel_difference_threshold_sensitivity', SCREENSHOT_COMPARISON_THRESHOLD_OPTIONS_DEFAULT)\n        try:\n            pixel_difference_threshold_sensitivity = int(pixel_difference_threshold_sensitivity)\n        except (ValueError, TypeError):\n            logger.warning(f\"Invalid pixel_difference_threshold_sensitivity value '{pixel_difference_threshold_sensitivity}', using default\")\n            pixel_difference_threshold_sensitivity = SCREENSHOT_COMPARISON_THRESHOLD_OPTIONS_DEFAULT\n\n\n        # Get minimum change percentage (per-watch > global > env var default)\n        # This controls what percentage of pixels must change to trigger a detection\n        min_change_percentage = processor_config.get('min_change_percentage')\n        if not min_change_percentage:\n            min_change_percentage = self.datastore.data['settings']['application'].get('min_change_percentage', 1)\n        try:\n            min_change_percentage = int(min_change_percentage)\n        except (ValueError, TypeError):\n            logger.warning(f\"Invalid min_change_percentage value '{min_change_percentage}', using default 0.1\")\n            min_change_percentage = 1\n\n        # Template matching for tracking content movement\n        template_matching_enabled = processor_config.get('auto_track_region', False) #@@todo disabled for now\n\n        if bounding_box:\n            try:\n                # Parse bounding box: \"x,y,width,height\"\n                parts = [int(p.strip()) for p in bounding_box.split(',')]\n                if len(parts) == 4:\n                    x, y, width, height = parts\n                    # Crop uses (left, top, right, bottom)\n                    crop_region = (max(0, x), max(0, y), x + width, y + height)\n                    logger.info(f\"UUID: {watch.get('uuid')} - Bounding box enabled: cropping to region {crop_region} (x={x}, y={y}, w={width}, h={height})\")\n                else:\n                    logger.warning(f\"UUID: {watch.get('uuid')} - Invalid bounding box format: {bounding_box} (expected 4 values)\")\n            except Exception as e:\n                logger.warning(f\"UUID: {watch.get('uuid')} - Failed to parse bounding box '{bounding_box}': {e}\")\n\n        # If no bounding box, check if visual selector (include_filters) is set for region-based comparison\n        if not crop_region:\n            include_filters = watch.get('include_filters', [])\n\n            if include_filters and len(include_filters) > 0:\n                # Get the first filter to use for cropping\n                first_filter = include_filters[0].strip()\n\n                if first_filter and self.xpath_data:\n                    try:\n                        import json\n                        # xpath_data is JSON string from browser\n                        xpath_data_obj = json.loads(self.xpath_data) if isinstance(self.xpath_data, str) else self.xpath_data\n\n                        # Find the bounding box for the first filter\n                        for element in xpath_data_obj.get('size_pos', []):\n                            # Match the filter with the element's xpath\n                            if element.get('xpath') == first_filter and element.get('highlight_as_custom_filter'):\n                                # Found the element - extract crop coordinates\n                                left = element.get('left', 0)\n                                top = element.get('top', 0)\n                                width = element.get('width', 0)\n                                height = element.get('height', 0)\n\n                                # Crop uses (left, top, right, bottom)\n                                crop_region = (max(0, left), max(0, top), left + width, top + height)\n\n                                logger.info(f\"UUID: {watch.get('uuid')} - Visual selector enabled: cropping to region {crop_region} for filter: {first_filter}\")\n                                break\n\n                    except Exception as e:\n                        logger.warning(f\"UUID: {watch.get('uuid')} - Failed to parse xpath_data for visual selector: {e}\")\n\n        # Store original crop region for template matching\n        original_crop_region = crop_region\n\n        # Check if this is the first check (no previous history)\n        history_keys = list(watch.history.keys())\n        if len(history_keys) == 0:\n            # First check - save baseline, no comparison\n            logger.info(f\"UUID: {watch.get('uuid')} - First check for watch {watch.get('uuid')} - saving baseline screenshot\")\n\n            # LibVIPS uses automatic reference counting - no explicit cleanup needed\n            update_obj = {\n                'previous_md5': hashlib.md5(self.screenshot).hexdigest(),\n                'last_error': False\n            }\n            logger.trace(f\"Processed in {time.time() - now:.3f}s\")\n            return False, update_obj, self.screenshot\n\n        # Get previous screenshot bytes from history\n        previous_timestamp = history_keys[-1]\n        previous_screenshot_bytes = watch.get_history_snapshot(timestamp=previous_timestamp)\n\n        # Screenshots are stored as PNG, so this should be bytes\n        if isinstance(previous_screenshot_bytes, str):\n            # If it's a string (shouldn't be for screenshots, but handle it)\n            previous_screenshot_bytes = previous_screenshot_bytes.encode('utf-8')\n\n        # Template matching is temporarily disabled pending OpenCV implementation\n        # crop_region calculated above will be used as-is\n\n        # Perform comparison in isolated subprocess to prevent memory leaks\n        try:\n            from .image_handler import isolated_opencv as process_screenshot_handler\n\n# stuff in watch doesnt need to be there\n            logger.debug(f\"UUID: {watch.get('uuid')} - Starting isolated subprocess comparison (crop_region={crop_region})\")\n\n            # Compare using isolated subprocess with OpenCV (async-safe to avoid blocking event loop)\n            # Pass raw bytes and crop region - subprocess handles all image operations\n            import asyncio\n            import threading\n\n            # Async-safe wrapper: runs coroutine in new thread with its own event loop\n            # This prevents blocking the async update worker's event loop\n            def run_async_in_thread():\n                return asyncio.run(\n                    process_screenshot_handler.compare_images_isolated(\n                        img_bytes_from=previous_screenshot_bytes,\n                        img_bytes_to=self.screenshot,\n                        pixel_difference_threshold=pixel_difference_threshold_sensitivity,\n                        blur_sigma=OPENCV_BLUR_SIGMA,\n                        crop_region=crop_region  # Pass crop region for isolated cropping\n                    )\n                )\n\n            # Run in thread to avoid blocking event loop when called from async update worker\n            result_container = [None]\n            exception_container = [None]\n\n            def thread_target():\n                try:\n                    result_container[0] = run_async_in_thread()\n                except Exception as e:\n                    exception_container[0] = e\n\n            thread = threading.Thread(target=thread_target, daemon=True, name=\"ImageDiff-Processor\")\n            thread.start()\n            thread.join(timeout=60)\n\n            if exception_container[0]:\n                raise exception_container[0]\n\n            # Subprocess returns only the change score - we decide if it's a \"change\"\n            change_score = result_container[0]\n            if change_score is None:\n                raise RuntimeError(\"Image comparison subprocess returned no result\")\n\n            changed_detected = change_score > min_change_percentage\n            logger.info(f\"UUID: {watch.get('uuid')} -  {process_screenshot_handler.IMPLEMENTATION_NAME}: {change_score:.2f}% pixels changed, pixel_diff_threshold_sensitivity: {pixel_difference_threshold_sensitivity:.0f} score={change_score:.2f}%, min_change_threshold={min_change_percentage}%\")\n\n        except Exception as e:\n            logger.error(f\"UUID: {watch.get('uuid')} - Failed to compare screenshots: {e}\")\n            logger.trace(f\"UUID: {watch.get('uuid')} - Processed in {time.time() - now:.3f}s\")\n\n            raise ProcessorException(\n                message=f\"UUID: {watch.get('uuid')} - Screenshot comparison failed: {e}\",\n                url=watch.get('url')\n            )\n\n        # Return results\n        update_obj = {\n            'previous_md5': hashlib.md5(self.screenshot).hexdigest(),\n            'last_error': False\n        }\n\n        if changed_detected:\n            logger.info(f\"UUID: {watch.get('uuid')} - Change detected using OpenCV! Score: {change_score:.2f}\")\n        else:\n            logger.debug(f\"UUID: {watch.get('uuid')} - No significant change using OpenCV. Score: {change_score:.2f}\")\n        logger.trace(f\"UUID: {watch.get('uuid')} - Processed in {time.time() - now:.3f}s\")\n\n        return changed_detected, update_obj, self.screenshot\n\n"
  },
  {
    "path": "changedetectionio/processors/image_ssim_diff/templates/image_ssim_diff/diff.html",
    "content": "{% extends 'base.html' %}\n{% from '_helpers.html' import render_field, render_checkbox_field, render_button %}\n\n{% block content %}\n\n\n\n
\n    
\n        
\n            {% if versions|length >= 1 %}\n                \n                \n                \n                \n                \n                \n                \n                \n            {% endif %}\n        
\n        
\n            \n                Change Detection: {{ \"%.2f\"|format(change_percentage) }}% of pixels changed\n                {% if change_percentage > 0.1 %}\n                    ⚠ Change Detected\n                {% else %}\n                    ✓ No Significant Change\n                {% endif %}\n            \n        
\n        {%- if versions|length >= 2 -%}\n            
\n                Keyboard: \n                 ← Previous\n                   → Next\n            
\n        {%- endif -%}\n    
\n
\n\n
\n    \n    
\n        \n        
\n            
Interactive Comparison
\n            
\n                Drag slider to compare Previous ({{ from_version|format_timestamp_timeago }})\n                vs Current ({{ to_version|format_timestamp_timeago }})\n            
\n            \n\n            
\n                \n                
\n                    \"Previous\n                
\n\n                \n                
\n                    \"Current\n                
\n\n                \n                
\n                    Previous\n                    Current\n                
\n\n                \n                
\n                    
\n                
\n            
\n        
\n\n        \n        
\n            
Difference Visualization
\n            
\n                Red = Changed Pixels\n            
\n            \n            \"Difference\n        
\n    
\n\n    {% if comparison_data and comparison_data.get('history') and comparison_data.history|length > 1 %}\n    
\n        
Comparison History
\n        
Recent comparison results (last {{ comparison_data.history|length }} checks)
\n        
\n            \n                \n                    \n                        \n                        \n                        \n                        \n                    \n                \n                \n                    {% for entry in comparison_data.history|reverse %}\n                    \n                        \n                        \n                        \n                        \n                    \n                    {% endfor %}\n                \n            
TimestampChange %MethodChanged?
{{ entry.timestamp|format_timestamp_timeago }}{{ \"%.2f\"|format(entry.change_percentage) }}%{{ entry.method }}\n                            {% if entry.changed %}\n                                Yes\n                            {% else %}\n                                No\n                            {% endif %}\n                        
\n        
\n    
\n    {% endif %}\n
\n\n\n\n\n\n{% endblock %}\n"
  },
  {
    "path": "changedetectionio/processors/image_ssim_diff/templates/image_ssim_diff/preview.html",
    "content": "{% extends 'base.html' %}\n\n{% block content %}\n\n    {% if versions|length >= 2 %}\n        
\n            
\n                
\n                     \n                    \n\n                
\n            
\n                
\n                Keyboard: \n                ← Previous   \n                → Next\n        
\n    {% endif %}\n\n    
\n        
Screenshot from {{ timestamp|format_timestamp_timeago }}
\n        \"Screenshot\n    
\n{% endblock %}\n"
  },
  {
    "path": "changedetectionio/processors/image_ssim_diff/util.py",
    "content": "\"\"\"\nDEPRECATED: All multiprocessing functions have been removed.\n\nThe image_ssim_diff processor now uses LibVIPS via ImageDiffHandler abstraction,\nwhich provides superior performance and memory efficiency through streaming\narchitecture and automatic threading.\n\nAll image operations are now handled by:\n- imagehandler.py: Abstract base class defining the interface\n- libvips_handler.py: LibVIPS implementation with streaming and threading\n\nHistorical note: This file previously contained multiprocessing workers for:\n- Template matching (find_region_with_template_matching_isolated)\n- Template regeneration (regenerate_template_isolated)\n- Image cropping (crop_image_isolated, crop_pil_image_isolated)\n\nThese have been replaced by handler methods which are:\n- Faster (no subprocess overhead)\n- More memory efficient (LibVIPS streaming)\n- Cleaner (no multiprocessing deadlocks)\n- Better tested (no logger/forking issues)\n\"\"\"\n"
  },
  {
    "path": "changedetectionio/processors/magic.py",
    "content": "\"\"\"\nContent Type Detection and Stream Classification\n\nThis module provides intelligent content-type detection for changedetection.io.\nIt addresses the common problem where HTTP Content-Type headers are missing, incorrect,\nor too generic, which would otherwise cause the wrong processor to be used.\n\nThe guess_stream_type class combines:\n1. HTTP Content-Type headers (when available and reliable)\n2. Python-magic library for MIME detection (analyzing actual file content)\n3. Content-based pattern matching for text formats (HTML tags, XML declarations, etc.)\n\nThis multi-layered approach ensures accurate detection of RSS feeds, JSON, HTML, PDF,\nplain text, CSV, YAML, and XML formats - even when servers provide misleading headers.\n\nUsed by: processors/text_json_diff/processor.py and other content processors\n\"\"\"\n\n# When to apply the 'cdata to real HTML' hack\nRSS_XML_CONTENT_TYPES = [\n    \"application/rss+xml\",\n    \"application/rdf+xml\",\n    \"application/atom+xml\",\n    \"text/rss+xml\",  # rare, non-standard\n    \"application/x-rss+xml\",  # legacy (older feed software)\n    \"application/x-atom+xml\",  # legacy (older Atom)\n]\n\n# JSON Content-types\nJSON_CONTENT_TYPES = [\n    \"application/activity+json\",\n    \"application/feed+json\",\n    \"application/json\",\n    \"application/ld+json\",\n    \"application/vnd.api+json\",\n]\n\n\n# Generic XML Content-types (non-RSS/Atom)\nXML_CONTENT_TYPES = [\n    \"text/xml\",\n    \"application/xml\",\n]\n\nHTML_PATTERNS = [' Union[float, None]:\n        # Clean and standardize the value (ie 1,400.00 should be 1400.00), even better would be store the whole thing as an integer.\n        standardized_value = raw_value\n\n        if ',' in standardized_value and '.' in standardized_value:\n            # Identify the correct decimal separator\n            if standardized_value.rfind('.') > standardized_value.rfind(','):\n                standardized_value = standardized_value.replace(',', '')\n            else:\n                standardized_value = standardized_value.replace('.', '').replace(',', '.')\n        else:\n            standardized_value = standardized_value.replace(',', '.')\n\n        # Remove any non-numeric characters except for the decimal point\n        standardized_value = re.sub(r'[^\\d.-]', '', standardized_value)\n\n        if standardized_value:\n            # Convert to float\n            # @todo locale needs to be the locale of the webpage\n            return float(parse_decimal(standardized_value, locale='en'))\n\n        return None\n\n    def __init__(self, *args, **kwargs):\n        # Define default values\n        default_values = {\n            'in_stock': None,\n            'price': None,\n            'currency': None,\n            'original_price': None\n        }\n\n        # Initialize the dictionary with default values\n        super().__init__(default_values)\n\n        # Update with any provided positional arguments (dictionaries)\n        if args:\n            if len(args) == 1 and isinstance(args[0], dict):\n                self.update(args[0])\n            else:\n                raise ValueError(\"Only one positional argument of type 'dict' is allowed\")\n\n    def __setitem__(self, key, value):\n        # Custom logic to handle setting price and original_price\n        if key == 'price' or key == 'original_price':\n            if isinstance(value, str):\n                value = self.parse_currency(raw_value=value)\n\n        super().__setitem__(key, value)\n\nclass Watch(BaseWatch):\n    def __init__(self, *arg, **kw):\n        super().__init__(*arg, **kw)\n        self['restock'] = Restock(kw['default']['restock']) if kw.get('default') and kw['default'].get('restock') else Restock()\n\n\n    def clear_watch(self):\n        super().clear_watch()\n        self.update({'restock': Restock()})\n\n    def extra_notification_token_values(self):\n        values = super().extra_notification_token_values()\n        values['restock'] = self.get('restock', {})\n        return values\n\n    def extra_notification_token_placeholder_info(self):\n        values = super().extra_notification_token_placeholder_info()\n\n        values.append(('restock.price', \"Price detected\"))\n        values.append(('restock.original_price', \"Original price at first check\"))\n\n        return values\n\n"
  },
  {
    "path": "changedetectionio/processors/restock_diff/api.yaml",
    "content": "components:\n  schemas:\n    processor_config_restock_diff:\n      type: object\n      description: Configuration for the restock_diff processor (restock and price tracking)\n      properties:\n        in_stock_processing:\n          type: string\n          enum: [in_stock_only, all_changes, 'off']\n          default: in_stock_only\n          description: |\n            When to trigger on stock changes:\n            - `in_stock_only`: Only trigger on Out Of Stock -> In Stock transitions\n            - `all_changes`: Trigger on any availability change\n            - `off`: Disable stock/availability tracking\n        follow_price_changes:\n          type: boolean\n          default: true\n          description: Monitor and track price changes\n        price_change_min:\n          type: [number, 'null']\n          description: Trigger a notification when the price drops below this value\n        price_change_max:\n          type: [number, 'null']\n          description: Trigger a notification when the price rises above this value\n        price_change_threshold_percent:\n          type: [number, 'null']\n          minimum: 0\n          maximum: 100\n          description: Minimum price change percentage since the original price to trigger a notification\n\npaths:\n  /watch:\n    post:\n      x-code-samples:\n        - lang: 'curl'\n          label: 'Restock & price tracking'\n          source: |\n            curl -X POST \"http://localhost:5000/api/v1/watch\" \\\n              -H \"x-api-key: YOUR_API_KEY\" \\\n              -H \"Content-Type: application/json\" \\\n              -d '{\n                \"url\": \"https://example.com/product\",\n                \"processor\": \"restock_diff\",\n                \"processor_config_restock_diff\": {\n                  \"in_stock_processing\": \"in_stock_only\",\n                  \"follow_price_changes\": true,\n                  \"price_change_threshold_percent\": 5\n                }\n              }'\n        - lang: 'Python'\n          label: 'Restock & price tracking'\n          source: |\n            import requests\n\n            headers = {\n                'x-api-key': 'YOUR_API_KEY',\n                'Content-Type': 'application/json'\n            }\n            data = {\n                'url': 'https://example.com/product',\n                'processor': 'restock_diff',\n                'processor_config_restock_diff': {\n                    'in_stock_processing': 'in_stock_only',\n                    'follow_price_changes': True,\n                    'price_change_threshold_percent': 5,\n                }\n            }\n            response = requests.post('http://localhost:5000/api/v1/watch',\n                                     headers=headers, json=data)\n            print(response.json())\n\n  /watch/{uuid}:\n    put:\n      x-code-samples:\n        - lang: 'curl'\n          label: 'Update restock config'\n          source: |\n            curl -X PUT \"http://localhost:5000/api/v1/watch/YOUR-UUID\" \\\n              -H \"x-api-key: YOUR_API_KEY\" \\\n              -H \"Content-Type: application/json\" \\\n              -d '{\n                \"processor_config_restock_diff\": {\n                  \"in_stock_processing\": \"all_changes\",\n                  \"follow_price_changes\": true,\n                  \"price_change_min\": 10.00,\n                  \"price_change_max\": 500.00\n                }\n              }'\n        - lang: 'Python'\n          label: 'Update restock config'\n          source: |\n            import requests\n\n            headers = {\n                'x-api-key': 'YOUR_API_KEY',\n                'Content-Type': 'application/json'\n            }\n            uuid = 'YOUR-UUID'\n            data = {\n                'processor_config_restock_diff': {\n                    'in_stock_processing': 'all_changes',\n                    'follow_price_changes': True,\n                    'price_change_min': 10.00,\n                    'price_change_max': 500.00,\n                }\n            }\n            response = requests.put(f'http://localhost:5000/api/v1/watch/{uuid}',\n                                    headers=headers, json=data)\n            print(response.text)\n\n  /tag/{uuid}:\n    put:\n      x-code-samples:\n        - lang: 'curl'\n          label: 'Set restock config on group/tag'\n          source: |\n            curl -X PUT \"http://localhost:5000/api/v1/tag/YOUR-TAG-UUID\" \\\n              -H \"x-api-key: YOUR_API_KEY\" \\\n              -H \"Content-Type: application/json\" \\\n              -d '{\n                \"overrides_watch\": true,\n                \"processor_config_restock_diff\": {\n                  \"in_stock_processing\": \"in_stock_only\",\n                  \"follow_price_changes\": true,\n                  \"price_change_threshold_percent\": 10\n                }\n              }'\n        - lang: 'Python'\n          label: 'Set restock config on group/tag'\n          source: |\n            import requests\n\n            headers = {\n                'x-api-key': 'YOUR_API_KEY',\n                'Content-Type': 'application/json'\n            }\n            tag_uuid = 'YOUR-TAG-UUID'\n            data = {\n                'overrides_watch': True,\n                'processor_config_restock_diff': {\n                    'in_stock_processing': 'in_stock_only',\n                    'follow_price_changes': True,\n                    'price_change_threshold_percent': 10,\n                }\n            }\n            response = requests.put(f'http://localhost:5000/api/v1/tag/{tag_uuid}',\n                                    headers=headers, json=data)\n            print(response.text)\n"
  },
  {
    "path": "changedetectionio/processors/restock_diff/forms.py",
    "content": "from wtforms import (\n    BooleanField,\n    validators,\n    FloatField\n)\nfrom wtforms.fields.choices import RadioField\nfrom wtforms.fields.form import FormField\nfrom wtforms.form import Form\nfrom flask_babel import lazy_gettext as _l\n\nfrom changedetectionio.forms import processor_text_json_diff_form\n\n\nclass RestockSettingsForm(Form):\n    in_stock_processing = RadioField(label=_l('Re-stock detection'), choices=[\n        ('in_stock_only', _l(\"In Stock only (Out Of Stock -> In Stock only)\")),\n        ('all_changes', _l(\"Any availability changes\")),\n        ('off', _l(\"Off, don't follow availability/restock\")),\n    ], default=\"in_stock_only\")\n\n    price_change_min = FloatField(_l('Below price to trigger notification'), [validators.Optional()],\n                                  render_kw={\"placeholder\": _l(\"No limit\"), \"size\": \"10\"})\n    price_change_max = FloatField(_l('Above price to trigger notification'), [validators.Optional()],\n                                  render_kw={\"placeholder\": _l(\"No limit\"), \"size\": \"10\"})\n    price_change_threshold_percent = FloatField(_l('Threshold in %% for price changes since the original price'), validators=[\n\n        validators.Optional(),\n        validators.NumberRange(min=0, max=100, message=_l(\"Should be between 0 and 100\")),\n    ], render_kw={\"placeholder\": \"0%\", \"size\": \"5\"})\n\n    follow_price_changes = BooleanField(_l('Follow price changes'), default=True)\n\nclass processor_settings_form(processor_text_json_diff_form):\n    processor_config_restock_diff = FormField(RestockSettingsForm)\n\n    def extra_tab_content(self):\n        return _l('Restock & Price Detection')\n\n    def extra_form_content(self):\n        output = \"\"\n\n        if getattr(self, 'watch', None) and getattr(self, 'datastore'):\n            for tag_uuid in self.watch.get('tags'):\n                tag = self.datastore.data['settings']['application']['tags'].get(tag_uuid, {})\n                if tag.get('overrides_watch'):\n                    # @todo - Quick and dirty, cant access 'url_for' here because its out of scope somehow\n                    output = f\"\"\"
Note! A Group tag overrides the restock and price detection here.
\"\"\"\n\n        output += \"\"\"\n        {% from '_helpers.html' import render_field, render_checkbox_field, render_button %}\n        \n\n        
\n            
\n                
\n                    {{ render_field(form.processor_config_restock_diff.in_stock_processing) }}\n                
\n                
\n                    {{ render_checkbox_field(form.processor_config_restock_diff.follow_price_changes) }}\n                    Changes in price should trigger a notification\n                
\n                
\n                    {{ render_field(form.processor_config_restock_diff.price_change_min, placeholder=watch.get('restock', {}).get('price')) }}\n                    Minimum amount, Trigger a change/notification when the price drops below this value.\n                
\n                
\n                    {{ render_field(form.processor_config_restock_diff.price_change_max, placeholder=watch.get('restock', {}).get('price')) }}\n                    Maximum amount, Trigger a change/notification when the price rises above this value.\n                
\n                
\n                    {{ render_field(form.processor_config_restock_diff.price_change_threshold_percent) }}\n                    Price must change more than this % to trigger a change since the first check.
\n                    For example, If the product is $1,000 USD originally, 2% would mean it has to change more than $20 since the first check.
\n                
\n            
\n        
\n        \"\"\"\n        return output"
  },
  {
    "path": "changedetectionio/processors/restock_diff/processor.py",
    "content": "from ..base import difference_detection_processor\nfrom ..exceptions import ProcessorException\nfrom . import Restock\nfrom loguru import logger\nfrom changedetectionio.content_fetchers.exceptions import checksumFromPreviousCheckWasTheSame\n\nimport urllib3\nimport time\n\nurllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)\n# Translatable strings - extracted by pybabel, translated at runtime in __init__.py\n# Use a marker function so pybabel can extract these strings\ndef _(x): return x  # Translation marker for extraction only\nname = _('Re-stock & Price detection for pages with a SINGLE product')\ndescription = _('Detects if the product goes back to in-stock')\ndel _  # Remove marker function\nprocessor_weight = 1\nlist_badge_text = \"Restock\"  # _()\n\nclass UnableToExtractRestockData(Exception):\n    def __init__(self, status_code):\n        # Set this so we can use it in other parts of the app\n        self.status_code = status_code\n        return\n\nclass MoreThanOnePriceFound(Exception):\n    def __init__(self):\n        return\n\ndef _search_prop_by_value(matches, value):\n    for properties in matches:\n        for prop in properties:\n            if value in prop[0]:\n                return prop[1]  # Yield the desired value and exit the function\n\ndef _deduplicate_prices(data):\n    import re\n\n    '''\n    Some price data has multiple entries, OR it has a single entry with ['$159', '159', 159, \"$ 159\"] or just \"159\"\n    Get all the values, clean it and add it to a set then return the unique values\n    '''\n    unique_data = set()\n\n    # Return the complete 'datum' where its price was not seen before\n    for datum in data:\n\n        if isinstance(datum.value, list):\n            # Process each item in the list\n            normalized_value = set([float(re.sub(r'[^\\d.]', '', str(item))) for item in datum.value if str(item).strip()])\n            unique_data.update(normalized_value)\n        else:\n            # Process single value\n            v = float(re.sub(r'[^\\d.]', '', str(datum.value)))\n            unique_data.add(v)\n\n    return list(unique_data)\n\n\n# =============================================================================\n# MEMORY MANAGEMENT: Why We Use Multiprocessing (Linux Only)\n# =============================================================================\n#\n# The get_itemprop_availability() function uses 'extruct' to parse HTML metadata\n# (JSON-LD, microdata, OpenGraph, etc). Extruct internally uses lxml, which wraps\n# libxml2 - a C library that allocates memory at the C level.\n#\n# Memory Leak Problem:\n# --------------------\n# 1. lxml's document_fromstring() creates thousands of Python objects backed by\n#    C-level allocations (nodes, attributes, text content)\n# 2. Python's garbage collector can mark these objects as collectible, but\n#    cannot force the OS to reclaim the actual C-level memory\n# 3. malloc/free typically doesn't return memory to OS - it just marks it as\n#    \"free in the process address space\"\n# 4. With repeated parsing of large HTML (5MB+ pages), memory accumulates even\n#    after Python GC runs\n#\n# Why Multiprocessing Fixes This:\n# --------------------------------\n# When a subprocess exits, the OS forcibly reclaims ALL memory including C-level\n# allocations that Python GC couldn't release. This ensures clean memory state\n# after each extraction.\n#\n# Performance Impact:\n# -------------------\n# - Memray analysis showed 1.2M document_fromstring allocations per page\n# - Without subprocess: memory grows by ~50-500MB per parse and lingers\n# - With subprocess: ~35MB overhead but forces full cleanup after each run\n# - Trade-off: 35MB resource_tracker vs 500MB+ accumulated leak = much better at scale\n#\n# References:\n# -----------\n# - lxml memory issues: https://medium.com/devopss-hole/python-lxml-memory-leak-b8d0b1000dc7\n# - libxml2 caching behavior: https://www.mail-archive.com/lxml@python.org/msg00026.html\n# - GC limitations with C extensions: https://benbernardblog.com/tracking-down-a-freaky-python-memory-leak-part-2/\n#\n# Additional Context:\n# -------------------\n# - jsonpath_ng (used to query the parsed data) is pure Python and doesn't leak\n# - The leak is specifically from lxml's document parsing, not the JSONPath queries\n# - Linux-only because multiprocessing spawn is well-tested there; other platforms\n#   use direct call as fallback\n#\n# Alternative Solution (Future Optimization):\n# -------------------------------------------\n# This entire problem could be avoided by using regex to extract just the machine\n# data blocks (JSON-LD, microdata, OpenGraph tags) BEFORE parsing with lxml:\n#\n#   1. Use regex to extract  blocks\n#   2. Use regex to extract  tags\n#   3. Use regex to find itemprop/itemtype attributes and their containing elements\n#   4. Parse ONLY those extracted snippets instead of the entire HTML document\n#\n# Benefits:\n#   - Avoids parsing 5MB of HTML when we only need a few KB of metadata\n#   - Eliminates the lxml memory leak entirely\n#   - Faster extraction (regex is much faster than DOM parsing)\n#   - No subprocess overhead needed\n#\n# Trade-offs:\n#   - Regex for HTML is brittle (comments, CDATA, edge cases)\n#   - Microdata extraction would be complex (need to track element boundaries)\n#   - Would need extensive testing to ensure we don't miss valid data\n#   - extruct is battle-tested; regex solution would need similar maturity\n#\n# For now, the subprocess approach is safer and leverages existing extruct code.\n# =============================================================================\n\n\ndef _extract_itemprop_availability_worker(pipe_conn):\n    \"\"\"\n    Subprocess worker for itemprop extraction (Linux memory management).\n\n    Uses spawn multiprocessing to isolate extruct/lxml memory allocations.\n    When the subprocess exits, the OS reclaims ALL memory including lxml's\n    C-level allocations that Python's GC cannot release.\n\n    Args:\n        pipe_conn: Pipe connection to receive HTML and send result\n    \"\"\"\n    import json\n    import gc\n\n    html_content = None\n    result_data = None\n\n    try:\n        # Receive HTML as raw bytes (no pickle)\n        html_bytes = pipe_conn.recv_bytes()\n        html_content = html_bytes.decode('utf-8')\n\n        # Explicitly delete html_bytes to free memory\n        del html_bytes\n        gc.collect()\n\n        # Perform extraction in subprocess (uses extruct/lxml)\n        result_data = get_itemprop_availability(html_content)\n\n        # Convert Restock object to dict for JSON serialization\n        result = {\n            'success': True,\n            'data': dict(result_data) if result_data else {}\n        }\n        pipe_conn.send_bytes(json.dumps(result).encode('utf-8'))\n\n        # Clean up before exit\n        del result_data, html_content, result\n        gc.collect()\n\n    except MoreThanOnePriceFound:\n        # Serialize the specific exception type\n        result = {\n            'success': False,\n            'exception_type': 'MoreThanOnePriceFound'\n        }\n        pipe_conn.send_bytes(json.dumps(result).encode('utf-8'))\n\n    except Exception as e:\n        # Serialize other exceptions\n        result = {\n            'success': False,\n            'exception_type': type(e).__name__,\n            'exception_message': str(e)\n        }\n        pipe_conn.send_bytes(json.dumps(result).encode('utf-8'))\n\n    finally:\n        # Final cleanup before subprocess exits\n        # Variables may already be deleted in try block, so use try/except\n        try:\n            del html_content\n        except (NameError, UnboundLocalError):\n            pass\n        try:\n            del result_data\n        except (NameError, UnboundLocalError):\n            pass\n        gc.collect()\n        pipe_conn.close()\n\n\ndef extract_itemprop_availability_safe(html_content) -> Restock:\n    \"\"\"\n    Extract itemprop availability with hybrid approach for memory efficiency.\n\n    Strategy (fastest to slowest, least to most memory):\n    1. Try pure Python extraction (JSON-LD, OpenGraph, microdata) - covers 80%+ of cases\n    2. Fall back to extruct with subprocess isolation on Linux for complex cases\n\n    Args:\n        html_content: HTML string to parse\n\n    Returns:\n        Restock: Extracted availability data\n\n    Raises:\n        MoreThanOnePriceFound: When multiple prices detected\n        Other exceptions: From extruct/parsing\n    \"\"\"\n    import platform\n\n    # Step 1: Try pure Python extraction first (fast, no lxml, no memory leak)\n    try:\n        from .pure_python_extractor import extract_metadata_pure_python, query_price_availability\n\n        logger.trace(\"Attempting pure Python metadata extraction (no lxml)\")\n        extracted_data = extract_metadata_pure_python(html_content)\n        price_data = query_price_availability(extracted_data)\n\n        # If we got price AND availability, we're done!\n        if price_data.get('price') and price_data.get('availability'):\n            result = Restock(price_data)\n            logger.debug(f\"Pure Python extraction successful: {dict(result)}\")\n            return result\n\n        # If we got some data but not everything, still try extruct for completeness\n        if price_data.get('price') or price_data.get('availability'):\n            logger.debug(f\"Pure Python extraction partial: {price_data}, will try extruct for completeness\")\n\n    except Exception as e:\n        logger.debug(f\"Pure Python extraction failed: {e}, falling back to extruct\")\n\n    # Step 2: Fall back to extruct (uses lxml, needs subprocess on Linux)\n    logger.trace(\"Falling back to extruct (lxml-based) with subprocess isolation\")\n\n    # Only use subprocess isolation on Linux\n    # Other platforms may have issues with spawn or don't need the aggressive memory management\n    if platform.system() == 'Linux':\n        import multiprocessing\n        import json\n        import gc\n\n        try:\n            ctx = multiprocessing.get_context('spawn')\n            parent_conn, child_conn = ctx.Pipe()\n            p = ctx.Process(target=_extract_itemprop_availability_worker, args=(child_conn,))\n            p.start()\n\n            # Send HTML as raw bytes (no pickle)\n            html_bytes = html_content.encode('utf-8')\n            parent_conn.send_bytes(html_bytes)\n\n            # Explicitly delete html_bytes copy immediately after sending\n            del html_bytes\n            gc.collect()\n\n            # Receive result as JSON\n            result_bytes = parent_conn.recv_bytes()\n            result = json.loads(result_bytes.decode('utf-8'))\n\n            # Wait for subprocess to complete\n            p.join()\n\n            # Close pipes\n            parent_conn.close()\n            child_conn.close()\n\n            # Clean up all subprocess-related objects\n            del p, parent_conn, child_conn, result_bytes\n            gc.collect()\n\n            # Handle result or re-raise exception\n            if result['success']:\n                # Reconstruct Restock object from dict\n                restock_obj = Restock(result['data'])\n                # Clean up result dict\n                del result\n                gc.collect()\n                return restock_obj\n            else:\n                # Re-raise the exception that occurred in subprocess\n                exception_type = result['exception_type']\n                exception_msg = result.get('exception_message', '')\n                del result\n                gc.collect()\n\n                if exception_type == 'MoreThanOnePriceFound':\n                    raise MoreThanOnePriceFound()\n                else:\n                    raise Exception(f\"{exception_type}: {exception_msg}\")\n\n        except Exception as e:\n            # If multiprocessing itself fails, log and fall back to direct call\n            logger.warning(f\"Subprocess extraction failed: {e}, falling back to direct call\")\n            gc.collect()\n            return get_itemprop_availability(html_content)\n    else:\n        # Non-Linux: direct call (no subprocess overhead needed)\n        return get_itemprop_availability(html_content)\n\n\n# should return Restock()\n# add casting?\ndef get_itemprop_availability(html_content) -> Restock:\n    \"\"\"\n    Kind of funny/cool way to find price/availability in one many different possibilities.\n    Use 'extruct' to find any possible RDFa/microdata/json-ld data, make a JSON string from the output then search it.\n    \"\"\"\n    from jsonpath_ng import parse\n\n    import re\n    now = time.time()\n    import extruct\n    logger.trace(f\"Imported extruct module in {time.time() - now:.3f}s\")\n\n    now = time.time()\n\n    # Extruct is very slow, I'm wondering if some ML is going to be faster (800ms on my i7), 'rdfa' seems to be the heaviest.\n    syntaxes = ['dublincore', 'json-ld', 'microdata', 'microformat', 'opengraph']\n    try:\n        data = extruct.extract(html_content, syntaxes=syntaxes)\n    except Exception as e:\n        logger.warning(f\"Unable to extract data, document parsing with extruct failed with {type(e).__name__} - {str(e)}\")\n        return Restock()\n\n    logger.trace(f\"Extruct basic extract of all metadata done in {time.time() - now:.3f}s\")\n\n    # First phase, dead simple scanning of anything that looks useful\n    value = Restock()\n    if data:\n        logger.debug(\"Using jsonpath to find price/availability/etc\")\n        price_parse = parse('$..(price|Price)')\n        pricecurrency_parse = parse('$..(pricecurrency|currency|priceCurrency )')\n        availability_parse = parse('$..(availability|Availability)')\n\n        price_result = _deduplicate_prices(price_parse.find(data))\n        if price_result:\n            # Right now, we just support single product items, maybe we will store the whole actual metadata seperately in teh future and\n            # parse that for the UI?\n            if len(price_result) > 1 and len(price_result) > 1:\n                # See of all prices are different, in the case that one product has many embedded data types with the same price\n                # One might have $121.95 and another 121.95 etc\n                logger.warning(f\"More than one price found {price_result}, throwing exception, cant use this plugin.\")\n                raise MoreThanOnePriceFound()\n\n            value['price'] = price_result[0]\n\n        pricecurrency_result = pricecurrency_parse.find(data)\n        if pricecurrency_result:\n            value['currency'] = pricecurrency_result[0].value\n\n        availability_result = availability_parse.find(data)\n        if availability_result:\n            value['availability'] = availability_result[0].value\n\n        if value.get('availability'):\n            value['availability'] = re.sub(r'(?i)^(https|http)://schema.org/', '',\n                                           value.get('availability').strip(' \"\\'').lower()) if value.get('availability') else None\n\n        # Second, go dig OpenGraph which is something that jsonpath_ng cant do because of the tuples and double-dots (:)\n        if not value.get('price') or value.get('availability'):\n            logger.debug(\"Alternatively digging through OpenGraph properties for restock/price info..\")\n            jsonpath_expr = parse('$..properties')\n\n            for match in jsonpath_expr.find(data):\n                if not value.get('price'):\n                    value['price'] = _search_prop_by_value([match.value], \"price:amount\")\n                if not value.get('availability'):\n                    value['availability'] = _search_prop_by_value([match.value], \"product:availability\")\n                if not value.get('currency'):\n                    value['currency'] = _search_prop_by_value([match.value], \"price:currency\")\n    logger.trace(f\"Processed with Extruct in {time.time()-now:.3f}s\")\n\n    return value\n\n\ndef is_between(number, lower=None, upper=None):\n    \"\"\"\n    Check if a number is between two values.\n\n    Parameters:\n    number (float): The number to check.\n    lower (float or None): The lower bound (inclusive). If None, no lower bound.\n    upper (float or None): The upper bound (inclusive). If None, no upper bound.\n\n    Returns:\n    bool: True if the number is between the lower and upper bounds, False otherwise.\n    \"\"\"\n    return (lower is None or lower <= number) and (upper is None or number <= upper)\n\n\nclass perform_site_check(difference_detection_processor):\n    screenshot = None\n    xpath_data = None\n\n    def run_changedetection(self, watch, force_reprocess=False):\n        import hashlib\n\n        if not watch:\n            raise Exception(\"Watch no longer exists.\")\n\n        current_raw_document_checksum = self.get_raw_document_checksum()\n        # Skip processing only if BOTH conditions are true:\n        # 1. HTML content unchanged (checksum matches last saved checksum)\n        # 2. Watch configuration was not edited (including trigger_text, filters, etc.)\n        # The was_edited flag handles all watch configuration changes, so we don't need\n        # separate checks for trigger_text or other processing rules.\n        if (not force_reprocess and\n            not watch.was_edited and\n            self.last_raw_content_checksum and\n            self.last_raw_content_checksum == current_raw_document_checksum):\n            raise checksumFromPreviousCheckWasTheSame()\n\n        # Unset any existing notification error\n        update_obj = {'last_notification_error': False, 'last_error': False, 'restock':  Restock()}\n\n        self.screenshot = self.fetcher.screenshot\n        self.xpath_data = self.fetcher.xpath_data\n\n        # Track the content type (readonly field, doesn't trigger was_edited)\n        update_obj['content-type'] = self.fetcher.headers.get('Content-Type', '')  # Use hyphen (matches OpenAPI spec)\n        update_obj[\"last_check_status\"] = self.fetcher.get_last_status_code()\n\n        # Save the raw content checksum to file (processor implementation detail, not watch config)\n        self.update_last_raw_content_checksum(current_raw_document_checksum)\n\n        # Only try to process restock information (like scraping for keywords) if the page was actually rendered correctly.\n        # Otherwise it will assume \"in stock\" because nothing suggesting the opposite was found\n#useless\n#        from ...html_tools import html_to_text\n#        text = html_to_text(self.fetcher.content)\n#        logger.debug(f\"Length of text after conversion: {len(text)}\")\n#        if not len(text):\n#            from ...content_fetchers.exceptions import ReplyWithContentButNoText\n#            raise ReplyWithContentButNoText(url=watch.link,\n#                                            status_code=self.fetcher.get_last_status_code(),\n#                                            screenshot=self.fetcher.screenshot,\n#                                            html_content=self.fetcher.content,\n#                                            xpath_data=self.fetcher.xpath_data\n#                                            )\n\n        # Which restock settings to compare against?\n        # Settings are stored in restock_diff.json (migrated from watch.json by update_30).\n        _extra_config = self.get_extra_watch_config('restock_diff.json')\n        restock_settings = _extra_config.get('restock_diff') or {\n            'follow_price_changes': True,\n            'in_stock_processing': 'in_stock_only',\n        }\n\n        # See if any tags have 'activate for individual watches in this tag/group?' enabled and use the first we find\n        for tag_uuid in watch.get('tags'):\n            tag = self.datastore.data['settings']['application']['tags'].get(tag_uuid, {})\n            if tag.get('overrides_watch'):\n                restock_settings = tag.get('processor_config_restock_diff') or {}\n                logger.info(f\"Watch {watch.get('uuid')} - Tag '{tag.get('title')}' selected for restock settings override\")\n                break\n\n\n        itemprop_availability = {}\n        multiple_prices_found = False\n\n        # Try built-in extraction first, this will scan metadata in the HTML\n        # On Linux, this runs in a subprocess to prevent lxml/extruct memory leaks\n        try:\n            itemprop_availability = extract_itemprop_availability_safe(self.fetcher.content)\n        except MoreThanOnePriceFound as e:\n            # Don't raise immediately - let plugins try to handle this case\n            # Plugins might be able to determine which price is correct\n            logger.warning(f\"Built-in detection found multiple prices on {watch.get('url')}, will try plugin override\")\n            multiple_prices_found = True\n            itemprop_availability = {}\n\n        # If built-in extraction didn't get both price AND availability, try plugin override\n        # Only check plugin if this watch is using a fetcher that might provide better data\n        has_price = itemprop_availability.get('price') is not None\n        has_availability = itemprop_availability.get('availability') is not None\n\n        # @TODO !!! some setting like \"Use as fallback\" or \"always use\", \"t\n        if not (has_price and has_availability) or True:\n            from changedetectionio.pluggy_interface import get_itemprop_availability_from_plugin\n            fetcher_name = watch.get('fetch_backend', 'html_requests')\n\n            # Resolve 'system' to the actual fetcher being used\n            # This allows plugins to work even when watch uses \"system settings default\"\n            if fetcher_name == 'system':\n                # Get the actual fetcher that was used (from self.fetcher)\n                # Fetcher class name gives us the actual backend (e.g., 'html_requests', 'html_webdriver')\n                actual_fetcher = type(self.fetcher).__name__\n                if 'html_requests' in actual_fetcher.lower():\n                    fetcher_name = 'html_requests'\n                elif 'webdriver' in actual_fetcher.lower() or 'playwright' in actual_fetcher.lower():\n                    fetcher_name = 'html_webdriver'\n                logger.debug(f\"Resolved 'system' fetcher to actual fetcher: {fetcher_name}\")\n\n            # Try plugin override - plugins can decide if they support this fetcher\n            if fetcher_name:\n                logger.debug(f\"Calling extra plugins for getting item price/availability (fetcher: {fetcher_name})\")\n                plugin_availability = get_itemprop_availability_from_plugin(self.fetcher.content, fetcher_name, self.fetcher, watch.link)\n\n                if plugin_availability:\n                    # Plugin provided better data, use it\n                    plugin_has_price = plugin_availability.get('price') is not None\n                    plugin_has_availability = plugin_availability.get('availability') is not None\n\n                    # Only use plugin data if it's actually better than what we have\n                    if plugin_has_price or plugin_has_availability:\n                        itemprop_availability = plugin_availability\n                        logger.info(f\"Using plugin-provided availability data for fetcher '{fetcher_name}' (built-in had price={has_price}, availability={has_availability}; plugin has price={plugin_has_price}, availability={plugin_has_availability})\")\n                if not plugin_availability:\n                    logger.debug(\"No item price/availability from plugins\")\n\n        # If we had multiple prices and plugins also failed, NOW raise the exception\n        if multiple_prices_found and not itemprop_availability.get('price'):\n            raise ProcessorException(\n                message=\"Cannot run, more than one price detected, this plugin is only for product pages with ONE product, try the content-change detection mode.\",\n                url=watch.get('url'),\n                status_code=self.fetcher.get_last_status_code(),\n                screenshot=self.fetcher.screenshot,\n                xpath_data=self.fetcher.xpath_data\n            )\n\n        # Something valid in get_itemprop_availability() by scraping metadata ?\n        if itemprop_availability.get('price') or itemprop_availability.get('availability'):\n            # Store for other usage\n            update_obj['restock'] = itemprop_availability\n\n            if itemprop_availability.get('availability'):\n                # @todo: Configurable?\n                if any(substring.lower() in itemprop_availability['availability'].lower() for substring in [\n                    'instock',\n                    'instoreonly',\n                    'limitedavailability',\n                    'onlineonly',\n                    'presale']\n                       ):\n                    update_obj['restock']['in_stock'] = True\n                else:\n                    update_obj['restock']['in_stock'] = False\n\n        # Main detection method\n        fetched_md5 = None\n\n        # store original price if not set\n        if itemprop_availability and itemprop_availability.get('price') and not itemprop_availability.get('original_price'):\n            itemprop_availability['original_price'] = itemprop_availability.get('price')\n            update_obj['restock'][\"original_price\"] = itemprop_availability.get('price')\n\n        if not self.fetcher.instock_data and not itemprop_availability.get('availability') and not itemprop_availability.get('price'):\n            raise ProcessorException(\n                message=f\"Unable to extract restock data for this page unfortunately. (Got code {self.fetcher.get_last_status_code()} from server), no embedded stock information was found and nothing interesting in the text, try using this watch with Chrome.\",\n                url=watch.get('url'),\n                status_code=self.fetcher.get_last_status_code(),\n                screenshot=self.fetcher.screenshot,\n                xpath_data=self.fetcher.xpath_data\n                )\n\n        logger.debug(f\"self.fetcher.instock_data is - '{self.fetcher.instock_data}' and itemprop_availability.get('availability') is {itemprop_availability.get('availability')}\")\n        # Nothing automatic in microdata found, revert to scraping the page\n        if self.fetcher.instock_data and itemprop_availability.get('availability') is None:\n            # 'Possibly in stock' comes from stock-not-in-stock.js when no string found above the fold.\n            # Careful! this does not really come from chrome/js when the watch is set to plaintext\n            update_obj['restock'][\"in_stock\"] = True if self.fetcher.instock_data == 'Possibly in stock' else False\n            logger.debug(f\"Watch UUID {watch.get('uuid')} restock check returned instock_data - '{self.fetcher.instock_data}' from JS scraper.\")\n\n        # Very often websites will lie about the 'availability' in the metadata, so if the scraped version says its NOT in stock, use that.\n        if self.fetcher.instock_data and self.fetcher.instock_data != 'Possibly in stock':\n            if update_obj['restock'].get('in_stock'):\n                logger.warning(\n                    f\"Lie detected in the availability machine data!! when scraping said its not in stock!! itemprop was '{itemprop_availability}' and scraped from browser was '{self.fetcher.instock_data}' update obj was {update_obj['restock']} \")\n                logger.warning(f\"Setting instock to FALSE, scraper found '{self.fetcher.instock_data}' in the body but metadata reported not-in-stock\")\n                update_obj['restock'][\"in_stock\"] = False\n\n        # What we store in the snapshot\n        price = update_obj.get('restock').get('price') if update_obj.get('restock').get('price') else \"\"\n        snapshot_content = f\"In Stock: {update_obj.get('restock').get('in_stock')} - Price: {price}\"\n\n        # Main detection method\n        fetched_md5 = hashlib.md5(snapshot_content.encode('utf-8')).hexdigest()\n\n        # The main thing that all this at the moment comes down to :)\n        changed_detected = False\n        logger.debug(f\"Watch UUID {watch.get('uuid')} restock check - Previous MD5: {watch.get('previous_md5')}, Fetched MD5 {fetched_md5}\")\n\n        # out of stock -> back in stock only?\n        if watch.get('restock') and watch['restock'].get('in_stock') != update_obj['restock'].get('in_stock'):\n            # Yes if we only care about it going to instock, AND we are in stock\n            if restock_settings.get('in_stock_processing') == 'in_stock_only' and update_obj['restock']['in_stock']:\n                changed_detected = True\n\n            if restock_settings.get('in_stock_processing') == 'all_changes':\n                # All cases\n                changed_detected = True\n\n        if restock_settings.get('follow_price_changes') and watch.get('restock') and update_obj.get('restock') and update_obj['restock'].get('price'):\n            price = float(update_obj['restock'].get('price'))\n            # Default to current price if no previous price found\n            if watch['restock'].get('original_price'):\n                previous_price = float(watch['restock'].get('original_price'))\n                # It was different, but negate it further down\n                if price != previous_price:\n                    changed_detected = True\n\n            # Minimum/maximum price limit\n            if update_obj.get('restock') and update_obj['restock'].get('price'):\n                logger.debug(\n                    f\"{watch.get('uuid')} - Change was detected, 'price_change_max' is '{restock_settings.get('price_change_max', '')}' 'price_change_min' is '{restock_settings.get('price_change_min', '')}', price from website is '{update_obj['restock'].get('price', '')}'.\")\n                if update_obj['restock'].get('price'):\n                    min_limit = float(restock_settings.get('price_change_min')) if restock_settings.get('price_change_min') else None\n                    max_limit = float(restock_settings.get('price_change_max')) if restock_settings.get('price_change_max') else None\n\n                    price = float(update_obj['restock'].get('price'))\n                    logger.debug(f\"{watch.get('uuid')} after float conversion - Min limit: '{min_limit}' Max limit: '{max_limit}' Price: '{price}'\")\n                    if min_limit or max_limit:\n                        if is_between(number=price, lower=min_limit, upper=max_limit):\n                            # Price was between min/max limit, so there was nothing todo in any case\n                            logger.trace(f\"{watch.get('uuid')} {price} is between {min_limit} and {max_limit}, nothing to check, forcing changed_detected = False (was {changed_detected})\")\n                            changed_detected = False\n                        else:\n                            logger.trace(f\"{watch.get('uuid')} {price} is between {min_limit} and {max_limit}, continuing normal comparison\")\n\n                    # Price comparison by %\n                    if watch['restock'].get('original_price') and changed_detected and restock_settings.get('price_change_threshold_percent'):\n                        previous_price = float(watch['restock'].get('original_price'))\n                        pc = float(restock_settings.get('price_change_threshold_percent'))\n                        change = abs((price - previous_price) / previous_price * 100)\n                        if change and change <= pc:\n                            logger.debug(f\"{watch.get('uuid')} Override change-detected to FALSE because % threshold ({pc}%) was {change:.3f}%\")\n                            changed_detected = False\n                        else:\n                            logger.debug(f\"{watch.get('uuid')} Price change was {change:.3f}% , (threshold {pc}%)\")\n\n        # Always record the new checksum\n        update_obj[\"previous_md5\"] = fetched_md5\n\n        return changed_detected, update_obj, snapshot_content.strip()\n"
  },
  {
    "path": "changedetectionio/processors/restock_diff/pure_python_extractor.py",
    "content": "\"\"\"\nPure Python metadata extractor - no lxml, no memory leaks.\n\nThis module provides a fast, memory-efficient alternative to extruct for common\ne-commerce metadata extraction. It handles:\n- JSON-LD (covers 80%+ of modern sites)\n- OpenGraph meta tags\n- Basic microdata attributes\n\nUses Python's built-in html.parser instead of lxml/libxml2, avoiding C-level\nmemory allocation issues. For edge cases, the main processor can fall back to\nextruct (with subprocess isolation on Linux).\n\"\"\"\n\nfrom html.parser import HTMLParser\nimport json\nimport re\nfrom loguru import logger\n\n\nclass JSONLDExtractor(HTMLParser):\n    \"\"\"\n    Extract JSON-LD structured data from HTML.\n\n    Finds all \n                \n            \n        `);\n    } else {\n        // Fallback: trigger download if popup is blocked\n        const a = document.createElement(\"a\");\n        a.href = jpegDataUrl;\n        a.download = \"changedetection-diff-\" + Date.now() + \".jpg\";\n        a.click();\n    }\n}\n\n/**\n * Update button UI state\n */\nfunction setButtonState(button, isLoading, originalHtml = '') {\n    if (!button) return;\n\n    if (isLoading) {\n        button.innerHTML = 'Generating...';\n        button.style.opacity = \"0.5\";\n        button.style.pointerEvents = \"none\";\n    } else {\n        button.innerHTML = originalHtml;\n        button.style.opacity = \"1\";\n        button.style.pointerEvents = \"auto\";\n    }\n}\n\n/**\n * Main function: Convert selected diff text to a shareable JPEG image\n *\n * Features:\n * - Expands partial selections to full lines\n * - Preserves all diff highlighting and formatting\n * - Adds metadata footer with URL and version info\n * - Embeds EXIF metadata in the JPEG\n * - Opens in new window or downloads if popup blocked\n */\nasync function diffToJpeg() {\n    // Validate dependencies\n    if (typeof html2canvas === 'undefined') {\n        alert(\"html2canvas library is not loaded yet. Please wait a moment and try again.\");\n        return;\n    }\n\n    // Validate selection\n    const selection = window.getSelection();\n    if (!selection || selection.rangeCount === 0 || selection.isCollapsed) {\n        alert(\"Please select the text/lines you want to capture first by highlighting with your mouse.\");\n        return;\n    }\n\n    const originalRange = selection.getRangeAt(0);\n    const differenceElement = document.getElementById(\"difference\");\n\n    if (!differenceElement || !differenceElement.contains(originalRange.commonAncestorContainer)) {\n        alert(\"Please select text within the diff content.\");\n        return;\n    }\n\n    // Setup UI state\n    const btn = document.getElementById(\"share-as-image-btn\");\n    const originalBtnHtml = btn ? btn.innerHTML : '';\n    setButtonState(btn, true);\n\n    let tempElement = null;\n\n    try {\n        // Expand selection to full lines and clone content\n        const expandedRange = expandRangeToFullLines(originalRange, differenceElement);\n        const selectedFragment = expandedRange.cloneContents();\n\n        // Count lines for footer\n        const selectedLines = countLines(selectedFragment);\n        const totalLines = countLines(differenceElement);\n\n        // Create temporary element with proper styling\n        tempElement = createCaptureElement(selectedFragment, differenceElement);\n        // Append footer to innerWrapper (inside the border), not outerWrapper\n        tempElement._innerWrapper.appendChild(createFooter(selectedLines, totalLines));\n\n        // Add to DOM for rendering\n        document.body.appendChild(tempElement);\n\n        // Wait for rendering\n        await new Promise(resolve => setTimeout(resolve, RENDER_DELAY_MS));\n\n        // Capture to canvas\n        const canvas = await html2canvas(tempElement, {\n            scale: CANVAS_SCALE,\n            useCORS: true,\n            allowTaint: true,\n            logging: false,\n            backgroundColor: '#ffffff',\n            scrollX: 0,\n            scrollY: 0\n        });\n\n        // Validate canvas\n        if (canvas.width === 0 || canvas.height === 0) {\n            throw new Error(\"Canvas is empty - no content captured\");\n        }\n\n        // Convert to JPEG\n        let jpeg = canvas.toDataURL(\"image/jpeg\", JPEG_QUALITY);\n\n        if (jpeg === \"data:,\" || jpeg.length < 100) {\n            throw new Error(\"Failed to generate image data\");\n        }\n\n        // Add EXIF metadata\n        jpeg = addExifMetadata(jpeg);\n\n        // Display the image\n        displayImage(jpeg);\n\n        // Clear selection\n        selection.removeAllRanges();\n\n    } catch (error) {\n        console.error(\"Error generating image:\", error);\n        alert(\"Failed to generate image: \" + error.message);\n    } finally {\n        // Cleanup\n        if (tempElement && tempElement.parentNode) {\n            tempElement.parentNode.removeChild(tempElement);\n        }\n        setButtonState(btn, false, originalBtnHtml);\n    }\n}\n"
  },
  {
    "path": "changedetectionio/static/js/stepper.js",
    "content": "$(document).ready(function(){\n   checkUserVal();\n   $('#fetch_backend input').on('change', checkUserVal);\n});\n\nvar checkUserVal = function(){\n    if($('#fetch_backend input:checked').val()=='html_requests') {\n      $('#request-override').show();\n      $('#webdriver-stepper').hide();\n    } else {\n      $('#request-override').hide();\n      $('#webdriver-stepper').show();\n    }\n};\n\n$('a.row-options').on('click', function(){\n    var row=$(this.closest('tr'));\n    switch($(this).data(\"action\")) {\n      case 'remove':\n        $(row).remove();\n      break;\n      case 'add':\n        var new_row=$(row).clone(true).insertAfter($(row));\n        $('input', new_new).val(\"\");\n      break;\n      case 'add':\n        var new_row=$(row).clone(true).insertAfter($(row));\n        $('input', new_new).val(\"\");\n      break;\n      case 'resend-step':\n\n      break;\n    }\n});"
  },
  {
    "path": "changedetectionio/static/js/tabs.js",
    "content": "// Rewrite this is a plugin.. is all this JS really 'worth it?'\n\nwindow.addEventListener('hashchange', function () {\n    // Only remove active from tab elements, not menu items\n    var tabs = document.querySelectorAll('.tabs li.active');\n    tabs.forEach(function(tab) {\n        tab.classList.remove('active');\n    });\n    document.body.classList.remove('full-width');\n    set_active_tab();\n}, false);\n\nvar has_errors = document.querySelectorAll(\".messages .error\");\nif (!has_errors.length) {\n    if (document.location.hash == \"\") {\n        location.replace(document.querySelector(\".tabs ul li:first-child a\").hash);\n    } else {\n        set_active_tab();\n    }\n} else {\n    focus_error_tab();\n}\n\nfunction set_active_tab() {\n    document.body.classList.remove('full-width');\n    var tab = document.querySelectorAll(\".tabs a[href='\" + location.hash + \"']\");\n    if (tab.length) {\n        tab[0].parentElement.classList.add(\"active\");\n    }\n}\n\nfunction focus_error_tab() {\n    // time to use jquery or vuejs really,\n    // activate the tab with the error\n    var tabs = document.querySelectorAll('.tabs li a'), i;\n    for (i = 0; i < tabs.length; ++i) {\n        var tab_name = tabs[i].hash.replace('#', '');\n        var pane_errors = document.querySelectorAll('#' + tab_name + ' .error')\n        if (pane_errors.length) {\n            document.location.hash = '#' + tab_name;\n            return true;\n        }\n    }\n    return false;\n}\n\n\n\n"
  },
  {
    "path": "changedetectionio/static/js/toast.js",
    "content": "/**\n * Toast - Modern toast notification system\n * Inspired by Toastify, Notyf, and React Hot Toast\n *\n * Usage:\n *   Toast.success('Operation completed!');\n *   Toast.error('Something went wrong');\n *   Toast.info('Here is some information');\n *   Toast.warning('Warning message');\n *   Toast.show('Custom message', { type: 'success', duration: 3000 });\n *\n * License: MIT\n */\n\n(function(window) {\n  'use strict';\n\n  // Toast configuration\n  const defaultConfig = {\n    duration: 5000,        // Auto-dismiss after 5 seconds (0 = no auto-dismiss)\n    position: 'top-center', // top-right, top-center, top-left, bottom-right, bottom-center, bottom-left\n    closeButton: true,     // Show close button\n    progressBar: true,     // Show progress bar\n    pauseOnHover: true,    // Pause auto-dismiss on hover\n    maxToasts: 5,          // Maximum toasts to show at once\n    offset: '20px',        // Offset from edge\n    zIndex: 10000,         // Z-index for toast container\n  };\n\n  let config = { ...defaultConfig };\n  let toastCount = 0;\n  let container = null;\n\n  /**\n   * Initialize toast system with custom config\n   */\n  function init(userConfig = {}) {\n    config = { ...defaultConfig, ...userConfig };\n    createContainer();\n  }\n\n  /**\n   * Create toast container if it doesn't exist\n   */\n  function createContainer() {\n    if (container) return;\n\n    container = document.createElement('div');\n    container.className = `toast-container toast-${config.position}`;\n    container.style.zIndex = config.zIndex;\n    document.body.appendChild(container);\n  }\n\n  /**\n   * Show a toast notification\n   */\n  function show(message, options = {}) {\n    createContainer();\n\n    const toast = createToastElement(message, options);\n\n    // Limit number of toasts\n    const existingToasts = container.querySelectorAll('.toast');\n    if (existingToasts.length >= config.maxToasts) {\n      removeToast(existingToasts[0]);\n    }\n\n    // Add to container\n    container.appendChild(toast);\n\n    // Trigger animation\n    requestAnimationFrame(() => {\n      toast.classList.add('toast-show');\n    });\n\n    // Auto-dismiss\n    if (options.duration !== 0 && (options.duration || config.duration) > 0) {\n      setupAutoDismiss(toast, options.duration || config.duration);\n    }\n\n    return {\n      dismiss: () => removeToast(toast)\n    };\n  }\n\n  /**\n   * Create toast DOM element\n   */\n  function createToastElement(message, options) {\n    const toast = document.createElement('div');\n    toast.className = `toast toast-${options.type || 'default'}`;\n    toast.setAttribute('role', 'alert');\n    toast.setAttribute('aria-live', 'polite');\n\n    // Icon\n    const icon = createIcon(options.type || 'default');\n    if (icon) {\n      toast.appendChild(icon);\n    }\n\n    // Message\n    const messageEl = document.createElement('div');\n    messageEl.className = 'toast-message';\n    messageEl.textContent = message;\n    toast.appendChild(messageEl);\n\n    // Close button\n    if (options.closeButton !== false && config.closeButton) {\n      const closeBtn = document.createElement('button');\n      closeBtn.className = 'toast-close';\n      closeBtn.innerHTML = '×';\n      closeBtn.setAttribute('aria-label', 'Close');\n      closeBtn.onclick = () => removeToast(toast);\n      toast.appendChild(closeBtn);\n    }\n\n    // Progress bar\n    if (options.progressBar !== false && config.progressBar && (options.duration || config.duration) > 0) {\n      const progressBar = document.createElement('div');\n      progressBar.className = 'toast-progress';\n      toast.appendChild(progressBar);\n      toast._progressBar = progressBar;\n    }\n\n    return toast;\n  }\n\n  /**\n   * Create icon based on toast type\n   */\n  function createIcon(type) {\n    const iconEl = document.createElement('div');\n    iconEl.className = 'toast-icon';\n\n    const svg = document.createElementNS('http://www.w3.org/2000/svg', 'svg');\n    svg.setAttribute('viewBox', '0 0 24 24');\n    svg.setAttribute('fill', 'none');\n    svg.setAttribute('stroke', 'currentColor');\n    svg.setAttribute('stroke-width', '2');\n\n    let path = '';\n    switch (type) {\n      case 'success':\n        path = 'M20 6L9 17l-5-5';\n        break;\n      case 'error':\n        path = 'M18 6L6 18M6 6l12 12';\n        break;\n      case 'warning':\n        path = 'M12 9v4m0 4h.01M12 2a10 10 0 100 20 10 10 0 000-20z';\n        svg.setAttribute('stroke-width', '1.5');\n        break;\n      case 'info':\n        path = 'M13 16h-1v-4h-1m1-4h.01M21 12a9 9 0 11-18 0 9 9 0 0118 0z';\n        svg.setAttribute('stroke-width', '1.5');\n        break;\n      default:\n        return null;\n    }\n\n    const pathEl = document.createElementNS('http://www.w3.org/2000/svg', 'path');\n    pathEl.setAttribute('d', path);\n    pathEl.setAttribute('stroke-linecap', 'round');\n    pathEl.setAttribute('stroke-linejoin', 'round');\n    svg.appendChild(pathEl);\n    iconEl.appendChild(svg);\n\n    return iconEl;\n  }\n\n  /**\n   * Setup auto-dismiss with progress bar\n   */\n  function setupAutoDismiss(toast, duration) {\n    let startTime = Date.now();\n    let remainingTime = duration;\n    let isPaused = false;\n    let animationFrame;\n\n    function updateProgress() {\n      if (isPaused) return;\n\n      const elapsed = Date.now() - startTime;\n      const progress = Math.min(elapsed / duration, 1);\n\n      if (toast._progressBar) {\n        toast._progressBar.style.transform = `scaleX(${1 - progress})`;\n      }\n\n      if (progress >= 1) {\n        removeToast(toast);\n      } else {\n        animationFrame = requestAnimationFrame(updateProgress);\n      }\n    }\n\n    // Pause on hover\n    if (config.pauseOnHover) {\n      toast.addEventListener('mouseenter', () => {\n        isPaused = true;\n        remainingTime = duration - (Date.now() - startTime);\n        cancelAnimationFrame(animationFrame);\n      });\n\n      toast.addEventListener('mouseleave', () => {\n        isPaused = false;\n        startTime = Date.now();\n        duration = remainingTime;\n        animationFrame = requestAnimationFrame(updateProgress);\n      });\n    }\n\n    animationFrame = requestAnimationFrame(updateProgress);\n  }\n\n  /**\n   * Remove toast with animation\n   */\n  function removeToast(toast) {\n    if (!toast || !toast.parentElement) return;\n\n    toast.classList.add('toast-hide');\n\n    // Remove after animation\n    setTimeout(() => {\n      if (toast.parentElement) {\n        toast.parentElement.removeChild(toast);\n      }\n    }, 300);\n  }\n\n  // Convenience methods\n  function success(message, options = {}) {\n    return show(message, { ...options, type: 'success' });\n  }\n\n  function error(message, options = {}) {\n    return show(message, { ...options, type: 'error' });\n  }\n\n  function warning(message, options = {}) {\n    return show(message, { ...options, type: 'warning' });\n  }\n\n  function info(message, options = {}) {\n    return show(message, { ...options, type: 'info' });\n  }\n\n  /**\n   * Clear all toasts\n   */\n  function clear() {\n    if (!container) return;\n    const toasts = container.querySelectorAll('.toast');\n    toasts.forEach(removeToast);\n  }\n\n  // Public API\n  window.Toast = {\n    init,\n    show,\n    success,\n    error,\n    warning,\n    info,\n    clear,\n    version: '1.0.0'\n  };\n\n  // Auto-initialize\n  document.addEventListener('DOMContentLoaded', () => {\n    init();\n  });\n\n})(window);\n"
  },
  {
    "path": "changedetectionio/static/js/toggle-theme.js",
    "content": "/**\n * @file\n * Toggles theme between light and dark mode.\n */\n$(document).ready(function () {\n\n    $(\".toggle-light-mode\").on(\"click\", function () {\n        const isDark = $(\"html\").attr(\"data-darkmode\") === \"true\";\n        $(\"html\").attr(\"data-darkmode\", !isDark);\n        setCookieValue(!isDark);\n    });\n\n    const setCookieValue = (value) => {\n        document.cookie = `css_dark_mode=${value};max-age=31536000;path=/`\n    }\n\n    // Search input box behaviour\n    const toggle_search = document.getElementById(\"toggle-search\");\n    const search_q = document.getElementById(\"search-q\");\n    if(search_q) {\n      window.addEventListener('keydown', function (e) {\n        if (e.altKey == true && e.keyCode == 83) {\n          search_q.classList.toggle('expanded');\n          search_q.focus();\n        }\n      });\n\n      search_q.onkeydown = (e) => {\n        var key = e.keyCode || e.which;\n        if (key === 13) {\n          document.searchForm.submit();\n        }\n      };\n      toggle_search.onclick = () => {\n        // Could be that they want to search something once text is in there\n        if (search_q.value.length) {\n          document.searchForm.submit();\n        } else {\n          // If not..\n          search_q.classList.toggle('expanded');\n          search_q.focus();\n        }\n      };\n    }\n\n    $('#heart-us').click(function () {\n        $(\"#overlay\").toggleClass('visible');\n        heartpath.style.fill = document.getElementById(\"overlay\").classList.contains(\"visible\") ? '#ff0000' : 'var(--color-background)';\n    });\n\n    setInterval(function () {\n        $('body').toggleClass('spinner-active', $.active > 0);\n    }, 2000);\n\n});\n"
  },
  {
    "path": "changedetectionio/static/js/vis.js",
    "content": "$(document).ready(function () {\n\n    // Lazy Hide/Show elements mechanism\n    $('[data-visible-for]').hide();\n    function show_related_elem(e) {\n        var n = $(e).attr('name') + \"=\" + $(e).val();\n        if (n === 'fetch_backend=system') {\n            n = \"fetch_backend=\" + default_system_fetch_backend;\n        }\n        $(`[data-visible-for~=\"${n}\"]`).show();\n    }\n    $(':radio').on('keyup keypress blur change click', function (e) {\n        $(`[data-visible-for]`).hide();\n        $('.advanced-options').hide();\n        show_related_elem(this);\n    });\n\n    $(':radio:checked').each(function (e) {\n       show_related_elem(this);\n    })\n\n\n    // Show advanced\n    $('.show-advanced').click(function (e) {\n        $(this).closest('.tab-pane-inner').find('.advanced-options').each(function (e) {\n            $(this).toggle();\n        })\n    });\n});"
  },
  {
    "path": "changedetectionio/static/js/visual-selector.js",
    "content": "// Copyright (C) 2021 Leigh Morresi (dgtlmoon@gmail.com)\n// All rights reserved.\n// yes - this is really a hack, if you are a front-ender and want to help, please get in touch!\n\nlet runInClearMode = false;\n\n$(document).ready(() => {\n    let currentSelections = [];\n    let currentSelection = null;\n    let appendToList = false;\n    let c, xctx, ctx;\n    let xScale = 1, yScale = 1;\n    let selectorImage, selectorImageRect, selectorData;\n    let elementHandlers = {}; // Store references to element selection handlers (needed for draw mode toggling)\n\n    // Box drawing mode variables (for image_ssim_diff processor)\n    let drawMode = false;\n    let isDrawing = false;\n    let isDragging = false;\n    let drawStartX, drawStartY;\n    let dragOffsetX, dragOffsetY;\n    let drawnBox = null;\n    let resizeHandle = null;\n    const HANDLE_SIZE = 8;\n    const isImageProcessor = $('input[value=\"image_ssim_diff\"]').is(':checked');\n\n\n    // Global jQuery selectors with \"Elem\" appended\n    const $selectorCanvasElem = $('#selector-canvas');\n    const $includeFiltersElem = $(\"#include_filters\");\n    const $selectorBackgroundElem = $(\"img#selector-background\");\n    const $selectorCurrentXpathElem = $(\"#selector-current-xpath span\");\n    const $fetchingUpdateNoticeElem = $('.fetching-update-notice');\n    const $selectorWrapperElem = $(\"#selector-wrapper\");\n\n    // Color constants\n    const FILL_STYLE_HIGHLIGHT = 'rgba(205,0,0,0.35)';\n    const FILL_STYLE_GREYED_OUT = 'rgba(205,205,205,0.95)';\n    const STROKE_STYLE_HIGHLIGHT = 'rgba(255,0,0, 0.9)';\n    const FILL_STYLE_REDLINE = 'rgba(255,0,0, 0.1)';\n    const STROKE_STYLE_REDLINE = 'rgba(225,0,0,0.9)';\n\n    $('#visualselector-tab').click(() => {\n        $selectorBackgroundElem.off('load');\n        currentSelections = [];\n        bootstrapVisualSelector();\n    });\n\n    function clearReset() {\n        ctx.clearRect(0, 0, c.width, c.height);\n\n        if ($includeFiltersElem.val().length) {\n            alert(\"Existing filters under the 'Filters & Triggers' tab were cleared.\");\n        }\n        $includeFiltersElem.val('');\n\n        currentSelections = [];\n\n        // Means we ignore the xpaths from the scraper marked as sel.highlight_as_custom_filter (it matched a previous selector)\n        runInClearMode = true;\n\n        highlightCurrentSelected();\n    }\n\n    function splitToList(v) {\n        return v.split('\\n').map(line => line.trim()).filter(line => line.length > 0);\n    }\n\n    function sortScrapedElementsBySize() {\n        // Sort the currentSelections array by area (width * height) in descending order\n        selectorData['size_pos'].sort((a, b) => {\n            const areaA = a.width * a.height;\n            const areaB = b.width * b.height;\n            return areaB - areaA;\n        });\n    }\n\n    $(document).on('keydown keyup', (event) => {\n        if (event.code === 'ShiftLeft' || event.code === 'ShiftRight') {\n            appendToList = event.type === 'keydown';\n        }\n\n        if (event.type === 'keydown') {\n            if ($selectorBackgroundElem.is(\":visible\") && event.key === \"Escape\") {\n                clearReset();\n            }\n        }\n    });\n\n    $('#clear-selector').on('click', () => {\n        clearReset();\n    });\n    // So if they start switching between visualSelector and manual filters, stop it from rendering old filters\n    $('li.tab a').on('click', () => {\n        runInClearMode = true;\n    });\n\n    if (!window.location.hash || window.location.hash !== '#visualselector') {\n        $selectorBackgroundElem.attr('src', '');\n        return;\n    }\n\n    bootstrapVisualSelector();\n\n    function bootstrapVisualSelector() {\n        $selectorBackgroundElem\n            .on(\"error\", () => {\n                $fetchingUpdateNoticeElem.html(\"Ooops! The VisualSelector tool needs at least one fetched page, please unpause the watch and/or wait for the watch to complete fetching and then reload this page.\")\n                    .css('color', '#bb0000');\n                $('#selector-current-xpath, #clear-selector').hide();\n            })\n            .on('load', () => {\n                console.log(\"Loaded background...\");\n                c = document.getElementById(\"selector-canvas\");\n                xctx = c.getContext(\"2d\");\n                ctx = c.getContext(\"2d\");\n                fetchData();\n                $selectorCanvasElem.off(\"mousemove mousedown\");\n            })\n            .attr(\"src\", screenshot_url);\n\n        let s = `${$selectorBackgroundElem.attr('src')}?${new Date().getTime()}`;\n        $selectorBackgroundElem.attr('src', s);\n    }\n\n    function alertIfFilterNotFound() {\n        let existingFilters = splitToList($includeFiltersElem.val());\n        let sizePosXpaths = selectorData['size_pos'].map(sel => sel.xpath);\n\n        for (let filter of existingFilters) {\n            if (!sizePosXpaths.includes(filter)) {\n                alert(`One or more of your existing filters was not found and will be removed when a new filter is selected.`);\n                break;\n            }\n        }\n    }\n\n    function fetchData() {\n        $fetchingUpdateNoticeElem.html(\"Fetching element data..\");\n\n        $.ajax({\n            url: watch_visual_selector_data_url,\n            context: document.body\n        }).done((data) => {\n            $fetchingUpdateNoticeElem.html(\"Rendering..\");\n            selectorData = data;\n\n            sortScrapedElementsBySize();\n            console.log(`Reported browser width from backend: ${data['browser_width']}`);\n\n            // Little sanity check for the user, alert them if something missing\n            alertIfFilterNotFound();\n\n            setScale();\n            reflowSelector();\n\n            // Initialize draw mode after everything is set up\n            initializeDrawMode();\n\n            $fetchingUpdateNoticeElem.fadeOut();\n        });\n    }\n\n    function updateFiltersText() {\n        // Assuming currentSelections is already defined and contains the selections\n        let uniqueSelections = new Set(currentSelections.map(sel => (sel[0] === '/' ? `xpath:${sel.xpath}` : sel.xpath)));\n\n        if (currentSelections.length > 0) {\n            // Convert the Set back to an array and join with newline characters\n            let textboxFilterText = Array.from(uniqueSelections).join(\"\\n\");\n            $includeFiltersElem.val(textboxFilterText);\n        }\n    }\n\n    function setScale() {\n        $selectorWrapperElem.show();\n        selectorImage = $selectorBackgroundElem[0];\n        selectorImageRect = selectorImage.getBoundingClientRect();\n\n        $selectorCanvasElem.attr({\n            'height': selectorImageRect.height,\n            'width': selectorImageRect.width\n        });\n        $selectorWrapperElem.attr('width', selectorImageRect.width);\n        $('#visual-selector-heading').css('max-width', selectorImageRect.width + \"px\")\n\n        xScale = selectorImageRect.width / selectorImage.naturalWidth;\n        yScale = selectorImageRect.height / selectorImage.naturalHeight;\n\n        ctx.strokeStyle = STROKE_STYLE_HIGHLIGHT;\n        ctx.fillStyle = FILL_STYLE_REDLINE;\n        ctx.lineWidth = 3;\n        console.log(\"Scaling set  x: \" + xScale + \" by y:\" + yScale);\n        $(\"#selector-current-xpath\").css('max-width', selectorImageRect.width);\n    }\n\n    function reflowSelector() {\n        $(window).resize(() => {\n            setScale();\n            highlightCurrentSelected();\n        });\n\n        setScale();\n\n        console.log(selectorData['size_pos'].length + \" selectors found\");\n\n        let existingFilters = splitToList($includeFiltersElem.val());\n\n        selectorData['size_pos'].forEach(sel => {\n            if ((!runInClearMode && sel.highlight_as_custom_filter) || existingFilters.includes(sel.xpath)) {\n                console.log(\"highlighting \" + c);\n                currentSelections.push(sel);\n            }\n        });\n\n\n        highlightCurrentSelected();\n        updateFiltersText();\n\n        // Store handler references for later use\n        elementHandlers.handleMouseMove = handleMouseMove.debounce(5);\n        elementHandlers.handleMouseDown = handleMouseDown.debounce(5);\n        elementHandlers.handleMouseLeave = highlightCurrentSelected.debounce(5);\n\n        $selectorCanvasElem.bind('mousemove', elementHandlers.handleMouseMove);\n        $selectorCanvasElem.bind('mousedown', elementHandlers.handleMouseDown);\n        $selectorCanvasElem.bind('mouseleave', elementHandlers.handleMouseLeave);\n\n        function handleMouseMove(e) {\n            if (!e.offsetX && !e.offsetY) {\n                const targetOffset = $(e.target).offset();\n                e.offsetX = e.pageX - targetOffset.left;\n                e.offsetY = e.pageY - targetOffset.top;\n            }\n\n            ctx.fillStyle = FILL_STYLE_HIGHLIGHT;\n\n            selectorData['size_pos'].forEach(sel => {\n                if (e.offsetY > sel.top * yScale && e.offsetY < sel.top * yScale + sel.height * yScale &&\n                    e.offsetX > sel.left * yScale && e.offsetX < sel.left * yScale + sel.width * yScale) {\n                    setCurrentSelectedText(sel.xpath);\n                    drawHighlight(sel);\n                    currentSelections.push(sel);\n                    currentSelection = sel;\n                    highlightCurrentSelected();\n                    currentSelections.pop();\n                }\n            })\n        }\n\n\n        function setCurrentSelectedText(s) {\n            $selectorCurrentXpathElem[0].innerHTML = s;\n        }\n\n        function drawHighlight(sel) {\n            ctx.strokeRect(sel.left * xScale, sel.top * yScale, sel.width * xScale, sel.height * yScale);\n            ctx.fillRect(sel.left * xScale, sel.top * yScale, sel.width * xScale, sel.height * yScale);\n        }\n\n        function handleMouseDown() {\n            // If we are in 'appendToList' mode, grow the list, if not, just 1\n            currentSelections = appendToList ? [...currentSelections, currentSelection] : [currentSelection];\n            highlightCurrentSelected();\n            updateFiltersText();\n        }\n\n    }\n\n    function highlightCurrentSelected() {\n        xctx.fillStyle = FILL_STYLE_GREYED_OUT;\n        xctx.strokeStyle = STROKE_STYLE_REDLINE;\n        xctx.lineWidth = 3;\n        xctx.clearRect(0, 0, c.width, c.height);\n\n        currentSelections.forEach(sel => {\n            //xctx.clearRect(sel.left * xScale, sel.top * yScale, sel.width * xScale, sel.height * yScale);\n            xctx.strokeRect(sel.left * xScale, sel.top * yScale, sel.width * xScale, sel.height * yScale);\n        });\n    }\n\n    // ============= BOX DRAWING MODE (for image_ssim_diff processor) =============\n\n    function initializeDrawMode() {\n        if (!isImageProcessor || !c) return;\n\n        const $selectorModeRadios = $('input[name=\"selector-mode\"]');\n        const $boundingBoxField = $('#bounding_box');\n        const $selectionModeField = $('#selection_mode');\n\n        // Load existing selection mode if present\n        const savedMode = $selectionModeField.val();\n        if (savedMode && (savedMode === 'element' || savedMode === 'draw')) {\n            $selectorModeRadios.filter(`[value=\"${savedMode}\"]`).prop('checked', true);\n            console.log('Loaded saved mode:', savedMode);\n        }\n\n        // Load existing bounding box if present\n        const existingBox = $boundingBoxField.val();\n        if (existingBox) {\n            try {\n                const parts = existingBox.split(',').map(p => parseFloat(p));\n                if (parts.length === 4) {\n                    drawnBox = {\n                        x: parts[0] * xScale,\n                        y: parts[1] * yScale,\n                        width: parts[2] * xScale,\n                        height: parts[3] * yScale\n                    };\n                    console.log('Loaded saved bounding box:', existingBox);\n                }\n            } catch (e) {\n                console.error('Failed to parse existing bounding box:', e);\n            }\n        }\n\n        // Update mode when radio changes\n        $selectorModeRadios.off('change').on('change', function() {\n            const newMode = $(this).val();\n            drawMode = newMode === 'draw';\n            console.log('Mode changed to:', newMode);\n\n            // Save the mode to the hidden field\n            $selectionModeField.val(newMode);\n\n            if (drawMode) {\n                enableDrawMode();\n            } else {\n                disableDrawMode();\n            }\n        });\n\n        // Set initial mode based on which radio is checked\n        drawMode = $selectorModeRadios.filter(':checked').val() === 'draw';\n        console.log('Initial mode:', drawMode ? 'draw' : 'element');\n\n        // Save initial mode\n        $selectionModeField.val(drawMode ? 'draw' : 'element');\n\n        if (drawMode) {\n            enableDrawMode();\n        }\n    }\n\n    function enableDrawMode() {\n        console.log('Enabling draw mode...');\n\n        // Unbind element selection handlers\n        $selectorCanvasElem.unbind('mousemove mousedown mouseleave');\n\n        // Set cursor to crosshair\n        $selectorCanvasElem.css('cursor', 'crosshair');\n\n        // Bind draw mode handlers\n        $selectorCanvasElem.on('mousedown', handleDrawMouseDown);\n        $selectorCanvasElem.on('mousemove', handleDrawMouseMove);\n        $selectorCanvasElem.on('mouseup', handleDrawMouseUp);\n        $selectorCanvasElem.on('mouseleave', handleDrawMouseUp);\n\n        // Clear element selections and xpath display\n        currentSelections = [];\n        $includeFiltersElem.val('');\n        $selectorCurrentXpathElem.html('Draw mode - click and drag to select an area');\n\n        // Clear the canvas\n        if (ctx && xctx) {\n            ctx.clearRect(0, 0, c.width, c.height);\n            xctx.clearRect(0, 0, c.width, c.height);\n        }\n\n        // Redraw if we have an existing box\n        if (drawnBox) {\n            drawBox();\n        }\n    }\n\n    function disableDrawMode() {\n        console.log('Disabling draw mode, switching to element mode...');\n\n        // Unbind draw handlers\n        $selectorCanvasElem.unbind('mousedown mousemove mouseup mouseleave');\n\n        // Reset cursor\n        $selectorCanvasElem.css('cursor', 'default');\n\n        // Clear drawn box\n        drawnBox = null;\n        $('#bounding_box').val('');\n\n        // Clear the canvases\n        if (ctx && xctx) {\n            ctx.clearRect(0, 0, c.width, c.height);\n            xctx.clearRect(0, 0, c.width, c.height);\n        }\n\n        // Restore element selections from include_filters\n        currentSelections = [];\n        if (selectorData && selectorData['size_pos']) {\n            let existingFilters = splitToList($includeFiltersElem.val());\n\n            selectorData['size_pos'].forEach(sel => {\n                if ((!runInClearMode && sel.highlight_as_custom_filter) || existingFilters.includes(sel.xpath)) {\n                    console.log(\"Restoring selection: \" + sel.xpath);\n                    currentSelections.push(sel);\n                }\n            });\n        }\n\n        // Re-enable element selection handlers using stored references\n        if (elementHandlers.handleMouseMove) {\n            $selectorCanvasElem.bind('mousemove', elementHandlers.handleMouseMove);\n            $selectorCanvasElem.bind('mousedown', elementHandlers.handleMouseDown);\n            $selectorCanvasElem.bind('mouseleave', elementHandlers.handleMouseLeave);\n        }\n\n        // Restore the element selection display\n        $selectorCurrentXpathElem.html('Hover over elements to select');\n\n        // Highlight the restored selections\n        highlightCurrentSelected();\n    }\n\n    function handleDrawMouseDown(e) {\n        const rect = c.getBoundingClientRect();\n        const x = e.clientX - rect.left;\n        const y = e.clientY - rect.top;\n\n        // Check if clicking on a resize handle\n        if (drawnBox) {\n            resizeHandle = getResizeHandle(x, y);\n            if (resizeHandle) {\n                isDrawing = true;\n                drawStartX = x;\n                drawStartY = y;\n                return;\n            }\n\n            // Check if clicking inside the box (for dragging)\n            if (isInsideBox(x, y)) {\n                isDragging = true;\n                dragOffsetX = x - drawnBox.x;\n                dragOffsetY = y - drawnBox.y;\n                $selectorCanvasElem.css('cursor', 'move');\n                return;\n            }\n        }\n\n        // Start new box\n        isDrawing = true;\n        drawStartX = x;\n        drawStartY = y;\n        drawnBox = { x: x, y: y, width: 0, height: 0 };\n    }\n\n    function handleDrawMouseMove(e) {\n        const rect = c.getBoundingClientRect();\n        const x = e.clientX - rect.left;\n        const y = e.clientY - rect.top;\n\n        // Update cursor based on position\n        if (!isDrawing && !isDragging && drawnBox) {\n            const handle = getResizeHandle(x, y);\n            if (handle) {\n                $selectorCanvasElem.css('cursor', getHandleCursor(handle));\n            } else if (isInsideBox(x, y)) {\n                $selectorCanvasElem.css('cursor', 'move');\n            } else {\n                $selectorCanvasElem.css('cursor', 'crosshair');\n            }\n        }\n\n        // Handle dragging the box\n        if (isDragging) {\n            drawnBox.x = x - dragOffsetX;\n            drawnBox.y = y - dragOffsetY;\n            drawBox();\n            return;\n        }\n\n        if (!isDrawing) return;\n\n        if (resizeHandle) {\n            // Resize existing box\n            resizeBox(x, y);\n        } else {\n            // Draw new box\n            drawnBox.width = x - drawStartX;\n            drawnBox.height = y - drawStartY;\n        }\n\n        drawBox();\n    }\n\n    function handleDrawMouseUp(e) {\n        if (!isDrawing && !isDragging) return;\n\n        isDrawing = false;\n        isDragging = false;\n        resizeHandle = null;\n\n        if (drawnBox) {\n            // Normalize box (handle negative dimensions)\n            if (drawnBox.width < 0) {\n                drawnBox.x += drawnBox.width;\n                drawnBox.width = Math.abs(drawnBox.width);\n            }\n            if (drawnBox.height < 0) {\n                drawnBox.y += drawnBox.height;\n                drawnBox.height = Math.abs(drawnBox.height);\n            }\n\n            // Constrain to canvas bounds\n            drawnBox.x = Math.max(0, Math.min(drawnBox.x, c.width - drawnBox.width));\n            drawnBox.y = Math.max(0, Math.min(drawnBox.y, c.height - drawnBox.height));\n\n            // Save to form field (convert from scaled to natural coordinates)\n            const naturalX = Math.round(drawnBox.x / xScale);\n            const naturalY = Math.round(drawnBox.y / yScale);\n            const naturalWidth = Math.round(drawnBox.width / xScale);\n            const naturalHeight = Math.round(drawnBox.height / yScale);\n\n            $('#bounding_box').val(`${naturalX},${naturalY},${naturalWidth},${naturalHeight}`);\n\n            drawBox();\n        }\n    }\n\n    function drawBox() {\n        if (!drawnBox) return;\n\n        // Clear and redraw\n        ctx.clearRect(0, 0, c.width, c.height);\n        xctx.clearRect(0, 0, c.width, c.height);\n\n        // Draw box\n        ctx.strokeStyle = STROKE_STYLE_REDLINE;\n        ctx.fillStyle = FILL_STYLE_REDLINE;\n        ctx.lineWidth = 3;\n\n        const drawX = drawnBox.width >= 0 ? drawnBox.x : drawnBox.x + drawnBox.width;\n        const drawY = drawnBox.height >= 0 ? drawnBox.y : drawnBox.y + drawnBox.height;\n        const drawW = Math.abs(drawnBox.width);\n        const drawH = Math.abs(drawnBox.height);\n\n        ctx.strokeRect(drawX, drawY, drawW, drawH);\n        ctx.fillRect(drawX, drawY, drawW, drawH);\n\n        // Draw resize handles\n        if (!isDrawing) {\n            drawResizeHandles(drawX, drawY, drawW, drawH);\n        }\n    }\n\n    function drawResizeHandles(x, y, w, h) {\n        ctx.fillStyle = '#fff';\n        ctx.strokeStyle = '#000';\n        ctx.lineWidth = 1;\n\n        const handles = [\n            { x: x, y: y },                    // top-left\n            { x: x + w, y: y },                // top-right\n            { x: x, y: y + h },                // bottom-left\n            { x: x + w, y: y + h }             // bottom-right\n        ];\n\n        handles.forEach(handle => {\n            ctx.fillRect(handle.x - HANDLE_SIZE/2, handle.y - HANDLE_SIZE/2, HANDLE_SIZE, HANDLE_SIZE);\n            ctx.strokeRect(handle.x - HANDLE_SIZE/2, handle.y - HANDLE_SIZE/2, HANDLE_SIZE, HANDLE_SIZE);\n        });\n    }\n\n    function isInsideBox(x, y) {\n        if (!drawnBox) return false;\n\n        const drawX = drawnBox.width >= 0 ? drawnBox.x : drawnBox.x + drawnBox.width;\n        const drawY = drawnBox.height >= 0 ? drawnBox.y : drawnBox.y + drawnBox.height;\n        const drawW = Math.abs(drawnBox.width);\n        const drawH = Math.abs(drawnBox.height);\n\n        return x >= drawX && x <= drawX + drawW && y >= drawY && y <= drawY + drawH;\n    }\n\n    function getResizeHandle(x, y) {\n        if (!drawnBox) return null;\n\n        const drawX = drawnBox.width >= 0 ? drawnBox.x : drawnBox.x + drawnBox.width;\n        const drawY = drawnBox.height >= 0 ? drawnBox.y : drawnBox.y + drawnBox.height;\n        const drawW = Math.abs(drawnBox.width);\n        const drawH = Math.abs(drawnBox.height);\n\n        const handles = {\n            'tl': { x: drawX, y: drawY },\n            'tr': { x: drawX + drawW, y: drawY },\n            'bl': { x: drawX, y: drawY + drawH },\n            'br': { x: drawX + drawW, y: drawY + drawH }\n        };\n\n        for (const [key, handle] of Object.entries(handles)) {\n            if (Math.abs(x - handle.x) <= HANDLE_SIZE && Math.abs(y - handle.y) <= HANDLE_SIZE) {\n                return key;\n            }\n        }\n\n        return null;\n    }\n\n    function getHandleCursor(handle) {\n        const cursors = {\n            'tl': 'nw-resize',\n            'tr': 'ne-resize',\n            'bl': 'sw-resize',\n            'br': 'se-resize'\n        };\n        return cursors[handle] || 'crosshair';\n    }\n\n    function resizeBox(x, y) {\n        const dx = x - drawStartX;\n        const dy = y - drawStartY;\n\n        const originalBox = { ...drawnBox };\n\n        switch (resizeHandle) {\n            case 'tl':\n                drawnBox.x = x;\n                drawnBox.y = y;\n                drawnBox.width = originalBox.x + originalBox.width - x;\n                drawnBox.height = originalBox.y + originalBox.height - y;\n                break;\n            case 'tr':\n                drawnBox.y = y;\n                drawnBox.width = x - originalBox.x;\n                drawnBox.height = originalBox.y + originalBox.height - y;\n                break;\n            case 'bl':\n                drawnBox.x = x;\n                drawnBox.width = originalBox.x + originalBox.width - x;\n                drawnBox.height = y - originalBox.y;\n                break;\n            case 'br':\n                drawnBox.width = x - originalBox.x;\n                drawnBox.height = y - originalBox.y;\n                break;\n        }\n\n        drawStartX = x;\n        drawStartY = y;\n    }\n});"
  },
  {
    "path": "changedetectionio/static/js/watch-overview.js",
    "content": "$(function () {\n    function normalizeUrl(el) {\n        const val = el.value.trim();\n        if (val && !/^[a-zA-Z][a-zA-Z\\d+\\-.]*:/.test(val)) {\n            el.value = 'https://' + val;\n        }\n    }\n\n    $('#url').on('blur keydown', function (e) {\n        if (e.type === 'blur' || e.key === 'Enter') {\n            normalizeUrl(this);\n        }\n    });\n\n    $('form').on('submit', function () {\n        normalizeUrl($('#url')[0]);\n    });\n\n    // Remove unviewed status when normally clicked\n    $('.diff-link').click(function () {\n        $(this).closest('.unviewed').removeClass('unviewed');\n    });\n\n    $('td[data-timestamp]').each(function () {\n        $(this).prop('title', new Intl.DateTimeFormat(undefined,\n            {\n                dateStyle: 'full',\n                timeStyle: 'long'\n            }).format($(this).data('timestamp') * 1000));\n    })\n\n    $(\"#checkbox-assign-tag\").click(function (e) {\n        $('#op_extradata').val(prompt(\"Enter a tag name\"));\n    });\n\n\n    $('.history-link').click(function (e) {\n        // Incase they click 'back' in the browser, it should be removed.\n        $(this).closest('tr').removeClass('unviewed');\n    });\n\n    $('.with-share-link > *').click(function () {\n        $(\"#copied-clipboard\").remove();\n\n        var range = document.createRange();\n        var n = $(\"#share-link\")[0];\n        range.selectNode(n);\n        window.getSelection().removeAllRanges();\n        window.getSelection().addRange(range);\n        document.execCommand(\"copy\");\n        window.getSelection().removeAllRanges();\n\n        $('.with-share-link').append('Copied to clipboard');\n        $(\"#copied-clipboard\").fadeOut(2500, function () {\n            $(this).remove();\n        });\n    });\n\n    $(\".watch-table tr\").click(function (event) {\n        var tagName = event.target.tagName.toLowerCase();\n        if (tagName === 'tr' || tagName === 'td') {\n            var x = $('input[type=checkbox]', this);\n            if (x) {\n                $(x).click();\n            }\n        }\n    });\n\n    // checkboxes - check all\n    $(\"#check-all\").click(function (e) {\n        $('input[type=checkbox]').not(this).prop('checked', this.checked);\n    });\n\n    const time_check_step_size_seconds=1;\n\n    // checkboxes - show/hide buttons\n    $(\"input[type=checkbox]\").click(function (e) {\n        if ($('input[type=checkbox]:checked').length) {\n            $('#checkbox-operations').slideDown();\n        } else {\n            $('#checkbox-operations').slideUp();\n        }\n    });\n\n    setInterval(function () {\n        // Background ETA completion for 'checking now'\n        $(\".watch-table .checking-now .last-checked\").each(function () {\n            const eta_complete = parseFloat($(this).data('eta_complete'));\n            const fetch_duration = parseInt($(this).data('fetchduration'));\n\n            if (eta_complete + 2 > nowtimeserver && fetch_duration > 3) {\n                const remaining_seconds = Math.abs(eta_complete) - nowtimeserver - 1;\n\n                let r = Math.round((1.0 - (remaining_seconds / fetch_duration)) * 100);\n                if (r < 10) {\n                    r = 10;\n                }\n                if (r >= 90) {\n                    r = 100;\n                }\n                $(this).css('background-size', `${r}% 100%`);\n            } else {\n                // Snap to full complete\n                $(this).css('background-size', `100% 100%`);\n            }\n        });\n\n        nowtimeserver = nowtimeserver + time_check_step_size_seconds;\n    }, time_check_step_size_seconds * 1000);\n});\n\n"
  },
  {
    "path": "changedetectionio/static/js/watch-settings.js",
    "content": "\nfunction request_textpreview_update() {\n    if (!$('body').hasClass('preview-text-enabled')) {\n        console.error(\"Preview text was requested but body tag was not setup\")\n        return\n    }\n\n    const data = {};\n    $('textarea:visible, input:visible').each(function () {\n        const $element = $(this); // Cache the jQuery object for the current element\n        const name = $element.attr('name'); // Get the name attribute of the element\n        data[name] = $element.is(':checkbox') ? ($element.is(':checked') ? $element.val() : false) : $element.val();\n    });\n\n    $('body').toggleClass('spinner-active', 1);\n\n    $.abortiveSingularAjax({\n        type: \"POST\",\n        url: preview_text_edit_filters_url,\n        data: data,\n        namespace: 'watchEdit'\n    }).done(function (data) {\n        console.debug(data['duration'])\n        $('#filters-and-triggers #text-preview-before-inner').text(data['before_filter']);\n        $('#filters-and-triggers #text-preview-inner')\n            .text(data['after_filter'])\n            .highlightLines([\n                {\n                    'color': 'var(--highlight-trigger-text-bg-color)',\n                    'lines': data['trigger_line_numbers'],\n                    'title': \"Triggers a change if this text appears, AND something changed in the document.\"\n                },\n                {\n                    'color': 'var(--highlight-ignored-text-bg-color)',\n                    'lines': data['ignore_line_numbers'],\n                    'title': \"Ignored for calculating changes, but still shown.\"\n                },\n                {\n                    'color': 'var(--highlight-blocked-text-bg-color)',\n                    'lines': data['blocked_line_numbers'],\n                    'title': \"No change-detection will occur because this text exists.\"\n                }\n            ])\n    }).fail(function (error) {\n        if (error.statusText === 'abort') {\n            console.log('Request was aborted due to a new request being fired.');\n        } else {\n            $('#filters-and-triggers #text-preview-inner').text('There was an error communicating with the server.');\n        }\n    })\n}\n\n\n$(document).ready(function () {\n\n    $('#notification-setting-reset-to-default').click(function (e) {\n        $('#notification_title').val('');\n        $('#notification_body').val('');\n        $('#notification_format').val('System default');\n        $('#notification_urls').val('');\n        $('#notification_muted_none').prop('checked', true); // in the case of a ternary field\n        e.preventDefault();\n    });\n    $(\"#notification-token-toggle\").click(function (e) {\n        e.preventDefault();\n        $('#notification-tokens-info').toggle();\n    });\n\n    toggleOpacity('#time_between_check_use_default', '#time-check-widget-wrapper, #time-between-check-schedule', false);\n\n\n    const vh = Math.max(document.documentElement.clientHeight || 0, window.innerHeight || 0);\n    $(\"#text-preview-inner\").css('max-height', (vh - 300) + \"px\");\n    $(\"#text-preview-before-inner\").css('max-height', (vh - 300) + \"px\");\n\n    $(\"#activate-text-preview\").click(function (e) {\n        $('body').toggleClass('preview-text-enabled')\n        request_textpreview_update();\n        const method = $('body').hasClass('preview-text-enabled') ? 'on' : 'off';\n        $('#filters-and-triggers textarea')[method]('blur', request_textpreview_update.throttle(1000));\n        $('#filters-and-triggers input')[method]('change', request_textpreview_update.throttle(1000));\n        $(\"#filters-and-triggers-tab\")[method]('click', request_textpreview_update.throttle(1000));\n    });\n    $('.minitabs-wrapper').miniTabs({\n        \"Content after filters\": \"#text-preview-inner\",\n        \"Content raw/before filters\": \"#text-preview-before-inner\"\n    });\n});\n\n"
  },
  {
    "path": "changedetectionio/static/styles/.dockerignore",
    "content": "node_modules\npackage-lock.json\n\n"
  },
  {
    "path": "changedetectionio/static/styles/.gitignore",
    "content": "node_modules\npackage-lock.json\n\n"
  },
  {
    "path": "changedetectionio/static/styles/diff-image.css",
    "content": ".comparison-score{padding:1em;background:var(--color-table-stripe);border-radius:4px;margin:1em 0;border:1px solid var(--color-border-table-cell);color:var(--color-text)}.change-detected{color:#d32f2f;font-weight:bold}.no-change{color:#388e3c;font-weight:bold}.comparison-grid{display:grid;grid-template-columns:1fr 1fr;gap:1em;margin:1em 1em}@media(max-width: 1200px){.comparison-grid{grid-template-columns:1fr}}.image-comparison{position:relative;width:100%;overflow:hidden;border:1px solid var(--color-border-table-cell);box-shadow:0 2px 4px rgba(0,0,0,.1);user-select:none}.image-comparison img{display:block;width:100%;height:auto;max-width:100%;border:none;box-shadow:none}.comparison-image-wrapper{position:relative;width:100%;display:flex;align-items:flex-start;justify-content:center;background-color:var(--color-background);background-image:linear-gradient(45deg, var(--color-table-stripe) 25%, transparent 25%),linear-gradient(-45deg, var(--color-table-stripe) 25%, transparent 25%),linear-gradient(45deg, transparent 75%, var(--color-table-stripe) 75%),linear-gradient(-45deg, transparent 75%, var(--color-table-stripe) 75%);background-size:20px 20px;background-position:0 0,0 10px,10px -10px,-10px 0px}.comparison-after{position:absolute;top:0;left:0;width:100%;height:100%;clip-path:inset(0 0 0 50%)}.comparison-slider{position:absolute;top:0;left:50%;width:4px;height:100%;background:#0078e7;cursor:ew-resize;transform:translateX(-2px);z-index:10}.comparison-handle{position:absolute;top:50%;left:50%;width:48px;height:48px;background:#0078e7;border:3px solid #fff;border-radius:50%;transform:translate(-50%, -50%);box-shadow:0 2px 8px rgba(0,0,0,.3);display:flex;align-items:center;justify-content:center;cursor:ew-resize;transition:top .1s ease-out}.comparison-handle::after{content:\"⇄\";color:#fff;font-size:24px;font-weight:bold;pointer-events:none}.comparison-labels{position:absolute;top:10px;width:100%;display:flex;justify-content:space-between;padding:0 0px;z-index:5;pointer-events:none}.comparison-label{background:rgba(0,0,0,.7);color:#fff;padding:.5em 1em;border-radius:4px;font-size:.9em;font-weight:bold}.screenshot-panel{text-align:center;background:var(--color-background);border:1px solid var(--color-border-table-cell);border-radius:4px;padding:1em;box-shadow:0 2px 4px rgba(0,0,0,.05)}.screenshot-panel h3{margin:0 0 1em 0;font-size:1.1em;color:var(--color-text);border-bottom:2px solid var(--color-background-button-primary);padding-bottom:.5em}.screenshot-panel.diff h3{border-bottom-color:#d32f2f}.screenshot-panel img{max-width:100%;height:auto;border:1px solid var(--color-border-table-cell);box-shadow:0 2px 4px rgba(0,0,0,.1)}.version-selector{display:inline-block;margin:0 .5em}.version-selector label{font-weight:bold;margin-right:.5em;color:var(--color-text)}#settings{background:var(--color-background);padding:1.5em;border-radius:4px;box-shadow:0 2px 4px rgba(0,0,0,.05);margin-bottom:2em;border:1px solid var(--color-border-table-cell)}#settings h2{margin-top:0;color:var(--color-text)}.diff-fieldset{border:none;padding:0;margin:0}.edit-link{float:right;margin-top:-0.5em}.comparison-description{color:var(--color-text-input-description);font-size:.9em;margin-bottom:1em}.download-link{color:var(--color-link);text-decoration:none;display:inline-flex;align-items:center;gap:.3em;font-size:.85em}.download-link:hover{text-decoration:underline}.diff-section-header{color:#d32f2f;font-size:.9em;margin-bottom:1em;font-weight:bold;display:flex;align-items:center;justify-content:center;gap:1em}.comparison-history-section{margin-top:3em;padding:1em;background:var(--color-background);border:1px solid var(--color-border-table-cell);border-radius:4px;box-shadow:0 2px 4px rgba(0,0,0,.05)}.comparison-history-section h3{color:var(--color-text)}.comparison-history-section p{color:var(--color-text-input-description);font-size:.9em}.history-changed-yes{color:#d32f2f;font-weight:bold}.history-changed-no{color:#388e3c}\n"
  },
  {
    "path": "changedetectionio/static/styles/diff.css",
    "content": "#diff-form{background:rgba(0,0,0,.05);padding:1em;border-radius:10px;margin-bottom:1em;color:#fff;font-size:.9rem;text-align:center}#diff-form label.from-to-label{width:4rem;text-decoration:none;padding:.5rem}#diff-form label.from-to-label#change-from{color:#b30000;background:#fadad7}#diff-form label.from-to-label#change-to{background:#eaf2c2;color:#406619}#diff-form #diff-style>span{display:inline-block;padding:.3em}#diff-form #diff-style>span label{font-weight:normal}#diff-form *{vertical-align:middle}body.difference-page section.content{padding-top:40px}#diff-ui{background:var(--color-background);padding:1rem;border-radius:5px}@media(min-width: 767px){#diff-ui{min-width:50%}}#diff-ui #text{font-size:11px}#diff-ui pre{white-space:break-spaces;overflow-wrap:anywhere}#diff-ui h1{display:inline;font-size:100%}#diff-ui #result{white-space:pre-wrap;word-break:break-word;overflow-wrap:break-word}#diff-ui .source{position:absolute;right:1%;top:.2em}@-moz-document url-prefix(){#diff-ui body{height:99%}}#diff-ui td#diff-col div{text-align:justify;white-space:pre-wrap}#diff-ui .ignored{background-color:#ccc;opacity:.7}#diff-ui .triggered{background-color:#1b98f8}#diff-ui .ignored.triggered{background-color:red}#diff-ui .tab-pane-inner#screenshot{text-align:center}#diff-ui .tab-pane-inner#screenshot img{max-width:99%}#diff-ui .pure-form button.reset-margin{margin:0px}#diff-ui .diff-fieldset{display:flex;align-items:center;gap:4px;flex-wrap:wrap}#diff-ui ul#highlightSnippetActions{list-style-type:none;display:flex;align-items:center;justify-content:center;gap:1.5rem;flex-wrap:wrap;padding:0;margin:0}#diff-ui ul#highlightSnippetActions li{display:flex;flex-direction:column;align-items:center;text-align:center;padding:.5rem;gap:.3rem}#diff-ui ul#highlightSnippetActions li button,#diff-ui ul#highlightSnippetActions li a{white-space:nowrap}#diff-ui ul#highlightSnippetActions span{font-size:.8rem;color:var(--color-text-input-description)}#diff-ui #cell-diff-jump-visualiser{display:flex;flex-direction:row;gap:1px;background:var(--color-background);border-radius:3px;overflow-x:hidden;position:sticky;top:0;z-index:10;padding-top:1rem;padding-bottom:1rem;justify-content:center}#diff-ui #cell-diff-jump-visualiser>div{flex:1;min-width:1px;max-width:10px;height:10px;background:var(--color-background-button-cancel);opacity:.3;border-radius:1px;transition:opacity .2s;position:relative}#diff-ui #cell-diff-jump-visualiser>div.deletion{background:#b30000;opacity:1}#diff-ui #cell-diff-jump-visualiser>div.insertion{background:#406619;opacity:1}#diff-ui #cell-diff-jump-visualiser>div.note{background:#406619;opacity:1}#diff-ui #cell-diff-jump-visualiser>div.mixed{background:linear-gradient(to right, #b30000 50%, #406619 50%);opacity:1}#diff-ui #cell-diff-jump-visualiser>div.current-position::after{content:\"\";position:absolute;bottom:-6px;left:50%;transform:translateX(-50%);width:0;height:0;border-left:4px solid rgba(0,0,0,0);border-right:4px solid rgba(0,0,0,0);border-bottom:4px solid var(--color-text)}#diff-ui #cell-diff-jump-visualiser>div:hover{opacity:.8;cursor:pointer}#text-diff-heading-area .snapshot-age{padding:4px;margin:.5rem 0;background-color:var(--color-background-snapshot-age);border-radius:3px;font-weight:bold;margin-bottom:4px}#text-diff-heading-area .snapshot-age.error{background-color:var(--color-error-background-snapshot-age);color:var(--color-error-text-snapshot-age)}#text-diff-heading-area .snapshot-age>*{padding-right:1rem}\n"
  },
  {
    "path": "changedetectionio/static/styles/package.json",
    "content": "{\n  \"name\": \"changedetection.io-theme\",\n  \"version\": \"0.0.3\",\n  \"description\": \"\",\n  \"main\": \"index.js\",\n  \"engines\": {\n    \"node\": \">=18.0.0\"\n  },\n  \"scripts\": {\n    \"watch\": \"sass --watch scss:. --style=compressed --no-source-map\",\n    \"build\": \"sass scss:. --style=compressed --no-source-map\"\n  },\n  \"author\": \"Leigh Morresi / Web Technologies s.r.o.\",\n  \"license\": \"Apache\",\n  \"dependencies\": {\n    \"sass\": \"^1.77.8\"\n  }\n}\n"
  },
  {
    "path": "changedetectionio/static/styles/pure-min.css",
    "content": "/*!\nPure v2.0.5\nCopyright 2013 Yahoo!\nLicensed under the BSD License.\nhttps://github.com/pure-css/pure/blob/master/LICENSE\n*/\n/*!\nnormalize.css v | MIT License | git.io/normalize\nCopyright (c) Nicolas Gallagher and Jonathan Neal\n*/\n/*! normalize.css v8.0.1 | MIT License | github.com/necolas/normalize.css */html{line-height:1.15;-webkit-text-size-adjust:100%}body{margin:0}main{display:block}h1{font-size:2em;margin:.67em 0}hr{-webkit-box-sizing:content-box;box-sizing:content-box;height:0;overflow:visible}pre{font-family:monospace,monospace;font-size:1em}a{background-color:transparent}abbr[title]{border-bottom:none;text-decoration:underline;-webkit-text-decoration:underline dotted;text-decoration:underline dotted}b,strong{font-weight:bolder}code,kbd,samp{font-family:monospace,monospace;font-size:1em}small{font-size:80%}sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}img{border-style:none}button,input,optgroup,select,textarea{font-family:inherit;font-size:100%;line-height:1.15;margin:0}button,input{overflow:visible}button,select{text-transform:none}[type=button],[type=reset],[type=submit],button{-webkit-appearance:button}[type=button]::-moz-focus-inner,[type=reset]::-moz-focus-inner,[type=submit]::-moz-focus-inner,button::-moz-focus-inner{border-style:none;padding:0}[type=button]:-moz-focusring,[type=reset]:-moz-focusring,[type=submit]:-moz-focusring,button:-moz-focusring{outline:1px dotted ButtonText}fieldset{padding:.35em .75em .625em}legend{-webkit-box-sizing:border-box;box-sizing:border-box;color:inherit;display:table;max-width:100%;padding:0;white-space:normal}progress{vertical-align:baseline}textarea{overflow:auto}[type=checkbox],[type=radio]{-webkit-box-sizing:border-box;box-sizing:border-box;padding:0}[type=number]::-webkit-inner-spin-button,[type=number]::-webkit-outer-spin-button{height:auto}[type=search]{-webkit-appearance:textfield;outline-offset:-2px}[type=search]::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{-webkit-appearance:button;font:inherit}details{display:block}summary{display:list-item}template{display:none}[hidden]{display:none}html{font-family:sans-serif}.hidden,[hidden]{display:none!important}.pure-img{max-width:100%;height:auto;display:block}.pure-g{letter-spacing:-.31em;text-rendering:optimizespeed;font-family:FreeSans,Arimo,\"Droid Sans\",Helvetica,Arial,sans-serif;display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-flow:row wrap;flex-flow:row wrap;-ms-flex-line-pack:start;align-content:flex-start}@media all and (-ms-high-contrast:none),(-ms-high-contrast:active){table .pure-g{display:block}}.opera-only :-o-prefocus,.pure-g{word-spacing:-.43em}.pure-u{display:inline-block;letter-spacing:normal;word-spacing:normal;vertical-align:top;text-rendering:auto}.pure-g [class*=pure-u]{font-family:sans-serif}.pure-u-1,.pure-u-1-1,.pure-u-1-12,.pure-u-1-2,.pure-u-1-24,.pure-u-1-3,.pure-u-1-4,.pure-u-1-5,.pure-u-1-6,.pure-u-1-8,.pure-u-10-24,.pure-u-11-12,.pure-u-11-24,.pure-u-12-24,.pure-u-13-24,.pure-u-14-24,.pure-u-15-24,.pure-u-16-24,.pure-u-17-24,.pure-u-18-24,.pure-u-19-24,.pure-u-2-24,.pure-u-2-3,.pure-u-2-5,.pure-u-20-24,.pure-u-21-24,.pure-u-22-24,.pure-u-23-24,.pure-u-24-24,.pure-u-3-24,.pure-u-3-4,.pure-u-3-5,.pure-u-3-8,.pure-u-4-24,.pure-u-4-5,.pure-u-5-12,.pure-u-5-24,.pure-u-5-5,.pure-u-5-6,.pure-u-5-8,.pure-u-6-24,.pure-u-7-12,.pure-u-7-24,.pure-u-7-8,.pure-u-8-24,.pure-u-9-24{display:inline-block;letter-spacing:normal;word-spacing:normal;vertical-align:top;text-rendering:auto}.pure-u-1-24{width:4.1667%}.pure-u-1-12,.pure-u-2-24{width:8.3333%}.pure-u-1-8,.pure-u-3-24{width:12.5%}.pure-u-1-6,.pure-u-4-24{width:16.6667%}.pure-u-1-5{width:20%}.pure-u-5-24{width:20.8333%}.pure-u-1-4,.pure-u-6-24{width:25%}.pure-u-7-24{width:29.1667%}.pure-u-1-3,.pure-u-8-24{width:33.3333%}.pure-u-3-8,.pure-u-9-24{width:37.5%}.pure-u-2-5{width:40%}.pure-u-10-24,.pure-u-5-12{width:41.6667%}.pure-u-11-24{width:45.8333%}.pure-u-1-2,.pure-u-12-24{width:50%}.pure-u-13-24{width:54.1667%}.pure-u-14-24,.pure-u-7-12{width:58.3333%}.pure-u-3-5{width:60%}.pure-u-15-24,.pure-u-5-8{width:62.5%}.pure-u-16-24,.pure-u-2-3{width:66.6667%}.pure-u-17-24{width:70.8333%}.pure-u-18-24,.pure-u-3-4{width:75%}.pure-u-19-24{width:79.1667%}.pure-u-4-5{width:80%}.pure-u-20-24,.pure-u-5-6{width:83.3333%}.pure-u-21-24,.pure-u-7-8{width:87.5%}.pure-u-11-12,.pure-u-22-24{width:91.6667%}.pure-u-23-24{width:95.8333%}.pure-u-1,.pure-u-1-1,.pure-u-24-24,.pure-u-5-5{width:100%}.pure-button{display:inline-block;line-height:normal;white-space:nowrap;vertical-align:middle;text-align:center;cursor:pointer;-webkit-user-drag:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;-webkit-box-sizing:border-box;box-sizing:border-box}.pure-button::-moz-focus-inner{padding:0;border:0}.pure-button-group{letter-spacing:-.31em;text-rendering:optimizespeed}.opera-only :-o-prefocus,.pure-button-group{word-spacing:-.43em}.pure-button-group .pure-button{letter-spacing:normal;word-spacing:normal;vertical-align:top;text-rendering:auto}.pure-button{font-family:inherit;font-size:100%;padding:.5em 1em;color:rgba(0,0,0,.8);border:none transparent;background-color:#e6e6e6;text-decoration:none;border-radius:2px}.pure-button-hover,.pure-button:focus,.pure-button:hover{background-image:-webkit-gradient(linear,left top,left bottom,from(transparent),color-stop(40%,rgba(0,0,0,.05)),to(rgba(0,0,0,.1)));background-image:linear-gradient(transparent,rgba(0,0,0,.05) 40%,rgba(0,0,0,.1))}.pure-button:focus{outline:0}.pure-button-active,.pure-button:active{-webkit-box-shadow:0 0 0 1px rgba(0,0,0,.15) inset,0 0 6px rgba(0,0,0,.2) inset;box-shadow:0 0 0 1px rgba(0,0,0,.15) inset,0 0 6px rgba(0,0,0,.2) inset;border-color:#000}.pure-button-disabled,.pure-button-disabled:active,.pure-button-disabled:focus,.pure-button-disabled:hover,.pure-button[disabled]{border:none;background-image:none;opacity:.4;cursor:not-allowed;-webkit-box-shadow:none;box-shadow:none;pointer-events:none}.pure-button-hidden{display:none}.pure-button-primary,.pure-button-selected,a.pure-button-primary,a.pure-button-selected{background-color:#0078e7;color:#fff}.pure-button-group .pure-button{margin:0;border-radius:0;border-right:1px solid rgba(0,0,0,.2)}.pure-button-group .pure-button:first-child{border-top-left-radius:2px;border-bottom-left-radius:2px}.pure-button-group .pure-button:last-child{border-top-right-radius:2px;border-bottom-right-radius:2px;border-right:none}.pure-form input[type=color],.pure-form input[type=date],.pure-form input[type=datetime-local],.pure-form input[type=datetime],.pure-form input[type=email],.pure-form input[type=month],.pure-form input[type=number],.pure-form input[type=password],.pure-form input[type=search],.pure-form input[type=tel],.pure-form input[type=text],.pure-form input[type=time],.pure-form input[type=url],.pure-form input[type=week],.pure-form select,.pure-form textarea{padding:.5em .6em;display:inline-block;border:1px solid #ccc;-webkit-box-shadow:inset 0 1px 3px #ddd;box-shadow:inset 0 1px 3px #ddd;border-radius:4px;vertical-align:middle;-webkit-box-sizing:border-box;box-sizing:border-box}.pure-form input:not([type]){padding:.5em .6em;display:inline-block;border:1px solid #ccc;-webkit-box-shadow:inset 0 1px 3px #ddd;box-shadow:inset 0 1px 3px #ddd;border-radius:4px;-webkit-box-sizing:border-box;box-sizing:border-box}.pure-form input[type=color]{padding:.2em .5em}.pure-form input[type=color]:focus,.pure-form input[type=date]:focus,.pure-form input[type=datetime-local]:focus,.pure-form input[type=datetime]:focus,.pure-form input[type=email]:focus,.pure-form input[type=month]:focus,.pure-form input[type=number]:focus,.pure-form input[type=password]:focus,.pure-form input[type=search]:focus,.pure-form input[type=tel]:focus,.pure-form input[type=text]:focus,.pure-form input[type=time]:focus,.pure-form input[type=url]:focus,.pure-form input[type=week]:focus,.pure-form select:focus,.pure-form textarea:focus{outline:0;border-color:#129fea}.pure-form input:not([type]):focus{outline:0;border-color:#129fea}.pure-form input[type=checkbox]:focus,.pure-form input[type=file]:focus,.pure-form input[type=radio]:focus{outline:thin solid #129fea;outline:1px auto #129fea}.pure-form .pure-checkbox,.pure-form .pure-radio{margin:.5em 0;display:block}.pure-form input[type=color][disabled],.pure-form input[type=date][disabled],.pure-form input[type=datetime-local][disabled],.pure-form input[type=datetime][disabled],.pure-form input[type=email][disabled],.pure-form input[type=month][disabled],.pure-form input[type=number][disabled],.pure-form input[type=password][disabled],.pure-form input[type=search][disabled],.pure-form input[type=tel][disabled],.pure-form input[type=text][disabled],.pure-form input[type=time][disabled],.pure-form input[type=url][disabled],.pure-form input[type=week][disabled],.pure-form select[disabled],.pure-form textarea[disabled]{cursor:not-allowed;background-color:#eaeded;color:#cad2d3}.pure-form input:not([type])[disabled]{cursor:not-allowed;background-color:#eaeded;color:#cad2d3}.pure-form input[readonly],.pure-form select[readonly],.pure-form textarea[readonly]{background-color:#eee;color:#777;border-color:#ccc}.pure-form input:focus:invalid,.pure-form select:focus:invalid,.pure-form textarea:focus:invalid{color:#b94a48;border-color:#e9322d}.pure-form input[type=checkbox]:focus:invalid:focus,.pure-form input[type=file]:focus:invalid:focus,.pure-form input[type=radio]:focus:invalid:focus{outline-color:#e9322d}.pure-form select{height:2.25em;border:1px solid #ccc;background-color:#fff}.pure-form select[multiple]{height:auto}.pure-form label{margin:.5em 0 .2em}.pure-form fieldset{margin:0;padding:.35em 0 .75em;border:0}.pure-form legend{display:block;width:100%;padding:.3em 0;margin-bottom:.3em;color:#333;border-bottom:1px solid #e5e5e5}.pure-form-stacked input[type=color],.pure-form-stacked input[type=date],.pure-form-stacked input[type=datetime-local],.pure-form-stacked input[type=datetime],.pure-form-stacked input[type=email],.pure-form-stacked input[type=file],.pure-form-stacked input[type=month],.pure-form-stacked input[type=number],.pure-form-stacked input[type=password],.pure-form-stacked input[type=search],.pure-form-stacked input[type=tel],.pure-form-stacked input[type=text],.pure-form-stacked input[type=time],.pure-form-stacked input[type=url],.pure-form-stacked input[type=week],.pure-form-stacked label,.pure-form-stacked select,.pure-form-stacked textarea{display:block;margin:.25em 0}.pure-form-stacked input:not([type]){display:block;margin:.25em 0}.pure-form-aligned input,.pure-form-aligned select,.pure-form-aligned textarea,.pure-form-message-inline{display:inline-block;vertical-align:middle}.pure-form-aligned textarea{vertical-align:top}.pure-form-aligned .pure-control-group{margin-bottom:.5em}.pure-form-aligned .pure-control-group label{text-align:right;display:inline-block;vertical-align:middle;width:10em;margin:0 1em 0 0}.pure-form-aligned .pure-controls{margin:1.5em 0 0 11em}.pure-form .pure-input-rounded,.pure-form input.pure-input-rounded{border-radius:2em;padding:.5em 1em}.pure-form .pure-group fieldset{margin-bottom:10px}.pure-form .pure-group input,.pure-form .pure-group textarea{display:block;padding:10px;margin:0 0 -1px;border-radius:0;position:relative;top:-1px}.pure-form .pure-group input:focus,.pure-form .pure-group textarea:focus{z-index:3}.pure-form .pure-group input:first-child,.pure-form .pure-group textarea:first-child{top:1px;border-radius:4px 4px 0 0;margin:0}.pure-form .pure-group input:first-child:last-child,.pure-form .pure-group textarea:first-child:last-child{top:1px;border-radius:4px;margin:0}.pure-form .pure-group input:last-child,.pure-form .pure-group textarea:last-child{top:-2px;border-radius:0 0 4px 4px;margin:0}.pure-form .pure-group button{margin:.35em 0}.pure-form .pure-input-1{width:100%}.pure-form .pure-input-3-4{width:75%}.pure-form .pure-input-2-3{width:66%}.pure-form .pure-input-1-2{width:50%}.pure-form .pure-input-1-3{width:33%}.pure-form .pure-input-1-4{width:25%}.pure-form-message-inline{display:inline-block;padding-left:.3em;color:#666;vertical-align:middle;font-size:.875em}.pure-form-message{display:block;color:#666;font-size:.875em}@media only screen and (max-width :480px){.pure-form button[type=submit]{margin:.7em 0 0}.pure-form input:not([type]),.pure-form input[type=color],.pure-form input[type=date],.pure-form input[type=datetime-local],.pure-form input[type=datetime],.pure-form input[type=email],.pure-form input[type=month],.pure-form input[type=number],.pure-form input[type=password],.pure-form input[type=search],.pure-form input[type=tel],.pure-form input[type=text],.pure-form input[type=time],.pure-form input[type=url],.pure-form input[type=week],.pure-form label{margin-bottom:.3em;display:block}.pure-group input:not([type]),.pure-group input[type=color],.pure-group input[type=date],.pure-group input[type=datetime-local],.pure-group input[type=datetime],.pure-group input[type=email],.pure-group input[type=month],.pure-group input[type=number],.pure-group input[type=password],.pure-group input[type=search],.pure-group input[type=tel],.pure-group input[type=text],.pure-group input[type=time],.pure-group input[type=url],.pure-group input[type=week]{margin-bottom:0}.pure-form-aligned .pure-control-group label{margin-bottom:.3em;text-align:left;display:block;width:100%}.pure-form-aligned .pure-controls{margin:1.5em 0 0 0}.pure-form-message,.pure-form-message-inline{display:block;font-size:.75em;padding:.2em 0 .8em}}.pure-menu{-webkit-box-sizing:border-box;box-sizing:border-box}.pure-menu-fixed{position:fixed;left:0;top:0;z-index:3}.pure-menu-item,.pure-menu-list{position:relative}.pure-menu-list{list-style:none;margin:0;padding:0}.pure-menu-item{padding:0;margin:0;height:100%}.pure-menu-heading,.pure-menu-link{display:block;text-decoration:none;white-space:nowrap}.pure-menu-horizontal{width:100%;white-space:nowrap}.pure-menu-horizontal .pure-menu-list{display:inline-block}.pure-menu-horizontal .pure-menu-heading,.pure-menu-horizontal .pure-menu-item,.pure-menu-horizontal .pure-menu-separator{display:inline-block;vertical-align:middle}.pure-menu-item .pure-menu-item{display:block}.pure-menu-children{display:none;position:absolute;left:100%;top:0;margin:0;padding:0;z-index:3}.pure-menu-horizontal .pure-menu-children{left:0;top:auto;width:inherit}.pure-menu-active>.pure-menu-children,.pure-menu-allow-hover:hover>.pure-menu-children{display:block;position:absolute}.pure-menu-has-children>.pure-menu-link:after{padding-left:.5em;content:\"\\25B8\";font-size:small}.pure-menu-horizontal .pure-menu-has-children>.pure-menu-link:after{content:\"\\25BE\"}.pure-menu-scrollable{overflow-y:scroll;overflow-x:hidden}.pure-menu-scrollable .pure-menu-list{display:block}.pure-menu-horizontal.pure-menu-scrollable .pure-menu-list{display:inline-block}.pure-menu-horizontal.pure-menu-scrollable{white-space:nowrap;overflow-y:hidden;overflow-x:auto;padding:.5em 0}.pure-menu-horizontal .pure-menu-children .pure-menu-separator,.pure-menu-separator{background-color:#ccc;height:1px;margin:.3em 0}.pure-menu-horizontal .pure-menu-separator{width:1px;height:1.3em;margin:0 .3em}.pure-menu-horizontal .pure-menu-children .pure-menu-separator{display:block;width:auto}.pure-menu-heading{text-transform:uppercase;color:#565d64}.pure-menu-link{color:#777}.pure-menu-children{background-color:#fff}.pure-menu-heading,.pure-menu-link{padding:.5em 1em}.pure-menu-disabled{opacity:.5}.pure-menu-disabled .pure-menu-link:hover{background-color:transparent;cursor:default}.pure-menu-active>.pure-menu-link,.pure-menu-link:focus,.pure-menu-link:hover{background-color:#eee}.pure-menu-selected>.pure-menu-link,.pure-menu-selected>.pure-menu-link:visited{color:#000}.pure-table{border-collapse:collapse;border-spacing:0;empty-cells:show;border:1px solid #cbcbcb}.pure-table caption{color:#000;font:italic 85%/1 arial,sans-serif;padding:1em 0;text-align:center}.pure-table td,.pure-table th{border-left:1px solid #cbcbcb;border-width:0 0 0 1px;font-size:inherit;margin:0;overflow:visible;padding:.5em 1em}.pure-table thead{background-color:#e0e0e0;color:#000;text-align:left;vertical-align:bottom}.pure-table td{background-color:transparent}.pure-table-odd td{background-color:#f2f2f2}.pure-table-striped tr:nth-child(2n-1) td{background-color:#f2f2f2}.pure-table-bordered td{border-bottom:1px solid #cbcbcb}.pure-table-bordered tbody>tr:last-child>td{border-bottom-width:0}.pure-table-horizontal td,.pure-table-horizontal th{border-width:0 0 1px 0;border-bottom:1px solid #cbcbcb}.pure-table-horizontal tbody>tr:last-child>td{border-bottom-width:0}"
  },
  {
    "path": "changedetectionio/static/styles/scss/_settings.scss",
    "content": "/**\n * SCSS variables (compile-time)\n * These can be used in media queries and other places where CSS custom properties don't work\n */\n\n// Breakpoints\n$desktop-wide-breakpoint: 980px;\n"
  },
  {
    "path": "changedetectionio/static/styles/scss/diff-image.scss",
    "content": "/**\n * Image Comparison Diff Styles\n * Styles for the interactive image comparison slider and screenshot diff visualization\n */\n\n.comparison-score {\n  padding: 1em;\n  background: var(--color-table-stripe);\n  border-radius: 4px;\n  margin: 1em 0;\n  border: 1px solid var(--color-border-table-cell);\n  color: var(--color-text);\n}\n\n.change-detected {\n  color: #d32f2f;\n  font-weight: bold;\n}\n\n.no-change {\n  color: #388e3c;\n  font-weight: bold;\n}\n\n.comparison-grid {\n  display: grid;\n  grid-template-columns: 1fr 1fr;\n  gap: 1em;\n  margin: 1em 1em;\n\n  @media (max-width: 1200px) {\n    grid-template-columns: 1fr;\n  }\n}\n\n/* Interactive Image Comparison Slider */\n.image-comparison {\n  position: relative;\n  width: 100%;\n  overflow: hidden;\n  border: 1px solid var(--color-border-table-cell);\n  box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);\n  user-select: none;\n\n  img {\n    display: block;\n    width: 100%;\n    height: auto;\n    max-width: 100%;\n    border: none;\n    box-shadow: none;\n  }\n}\n\n/* Image wrappers with checkered background */\n.comparison-image-wrapper {\n  position: relative;\n  width: 100%;\n  display: flex;\n  align-items: flex-start;\n  justify-content: center;\n  /* Very light checkered background pattern */\n  background-color: var(--color-background);\n  background-image:\n    linear-gradient(45deg, var(--color-table-stripe) 25%, transparent 25%),\n    linear-gradient(-45deg, var(--color-table-stripe) 25%, transparent 25%),\n    linear-gradient(45deg, transparent 75%, var(--color-table-stripe) 75%),\n    linear-gradient(-45deg, transparent 75%, var(--color-table-stripe) 75%);\n  background-size: 20px 20px;\n  background-position: 0 0, 0 10px, 10px -10px, -10px 0px;\n}\n\n.comparison-after {\n  position: absolute;\n  top: 0;\n  left: 0;\n  width: 100%;\n  height: 100%;\n  clip-path: inset(0 0 0 50%);\n}\n\n.comparison-slider {\n  position: absolute;\n  top: 0;\n  left: 50%;\n  width: 4px;\n  height: 100%;\n  background: #0078e7;\n  cursor: ew-resize;\n  transform: translateX(-2px);\n  z-index: 10;\n}\n\n.comparison-handle {\n  position: absolute;\n  top: 50%;\n  left: 50%;\n  width: 48px;\n  height: 48px;\n  background: #0078e7;\n  border: 3px solid white;\n  border-radius: 50%;\n  transform: translate(-50%, -50%);\n  box-shadow: 0 2px 8px rgba(0, 0, 0, 0.3);\n  display: flex;\n  align-items: center;\n  justify-content: center;\n  cursor: ew-resize;\n  transition: top 0.1s ease-out;\n\n  &::after {\n    content: '⇄';\n    color: white;\n    font-size: 24px;\n    font-weight: bold;\n    pointer-events: none;\n  }\n}\n\n.comparison-labels {\n  position: absolute;\n  top: 10px;\n  width: 100%;\n  display: flex;\n  justify-content: space-between;\n  padding: 0 0px;\n  z-index: 5;\n  pointer-events: none;\n}\n\n.comparison-label {\n  background: rgba(0, 0, 0, 0.7);\n  color: white;\n  padding: 0.5em 1em;\n  border-radius: 4px;\n  font-size: 0.9em;\n  font-weight: bold;\n}\n\n.screenshot-panel {\n  text-align: center;\n  background: var(--color-background);\n  border: 1px solid var(--color-border-table-cell);\n  border-radius: 4px;\n  padding: 1em;\n  box-shadow: 0 2px 4px rgba(0, 0, 0, 0.05);\n\n  h3 {\n    margin: 0 0 1em 0;\n    font-size: 1.1em;\n    color: var(--color-text);\n    border-bottom: 2px solid var(--color-background-button-primary);\n    padding-bottom: 0.5em;\n  }\n\n  &.diff h3 {\n    border-bottom-color: #d32f2f;\n  }\n\n  img {\n    max-width: 100%;\n    height: auto;\n    border: 1px solid var(--color-border-table-cell);\n    box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);\n  }\n}\n\n.version-selector {\n  display: inline-block;\n  margin: 0 0.5em;\n\n  label {\n    font-weight: bold;\n    margin-right: 0.5em;\n    color: var(--color-text);\n  }\n}\n\n#settings {\n  background: var(--color-background);\n  padding: 1.5em;\n  border-radius: 4px;\n  box-shadow: 0 2px 4px rgba(0, 0, 0, 0.05);\n  margin-bottom: 2em;\n  border: 1px solid var(--color-border-table-cell);\n\n  h2 {\n    margin-top: 0;\n    color: var(--color-text);\n  }\n}\n\n.diff-fieldset {\n  border: none;\n  padding: 0;\n  margin: 0;\n}\n\n.edit-link {\n  float: right;\n  margin-top: -0.5em;\n}\n\n.comparison-description {\n  color: var(--color-text-input-description);\n  font-size: 0.9em;\n  margin-bottom: 1em;\n}\n\n.download-link {\n  color: var(--color-link);\n  text-decoration: none;\n  display: inline-flex;\n  align-items: center;\n  gap: 0.3em;\n  font-size: 0.85em;\n\n  &:hover {\n    text-decoration: underline;\n  }\n}\n\n.diff-section-header {\n  color: #d32f2f;\n  font-size: 0.9em;\n  margin-bottom: 1em;\n  font-weight: bold;\n  display: flex;\n  align-items: center;\n  justify-content: center;\n  gap: 1em;\n}\n\n.comparison-history-section {\n  margin-top: 3em;\n  padding: 1em;\n  background: var(--color-background);\n  border: 1px solid var(--color-border-table-cell);\n  border-radius: 4px;\n  box-shadow: 0 2px 4px rgba(0, 0, 0, 0.05);\n\n  h3 {\n    color: var(--color-text);\n  }\n\n  p {\n    color: var(--color-text-input-description);\n    font-size: 0.9em;\n  }\n}\n\n.history-changed-yes {\n  color: #d32f2f;\n  font-weight: bold;\n}\n\n.history-changed-no {\n  color: #388e3c;\n}\n"
  },
  {
    "path": "changedetectionio/static/styles/scss/diff.scss",
    "content": "#diff-form {\n\n  background: rgba(0, 0, 0, .05);\n  padding: 1em;\n  border-radius: 10px;\n  margin-bottom: 1em;\n  color: #fff;\n  font-size: 0.9rem;\n  text-align: center;\n\n  label.from-to-label {\n    width: 4rem;\n    text-decoration: none;\n    padding: 0.5rem;\n\n    &#change-from {\n      color: #b30000;\n      background: #fadad7\n    }\n\n    &#change-to {\n      background: #eaf2c2;\n      color: #406619;\n    }\n  }\n\n  #diff-style {\n    >span {\n      display: inline-block;\n      padding: 0.3em;\n      label {\n        font-weight: normal;\n      }\n    }\n  }\n  * {\n    vertical-align: middle;\n  }\n\n}\n\nbody.difference-page {\n  section.content {\n    padding-top: 40px;\n  }\n}\n\n#diff-ui {\n\n  background: var(--color-background);\n  padding: 1rem;\n  border-radius: 5px;\n\n  @media (min-width: 767px) {\n    min-width: 50%;\n  }\n\n  // The first tab 'text' diff\n  #text {\n    font-size: 11px;\n  }\n\n  pre {\n    white-space: break-spaces;\n    overflow-wrap: anywhere;\n  }\n\n\n  h1 {\n    display: inline;\n    font-size: 100%;\n  }\n\n  #result {\n    white-space: pre-wrap;\n    word-break: break-word;\n    overflow-wrap: break-word;\n  }\n\n\n  .source {\n    position: absolute;\n    right: 1%;\n    top: .2em;\n  }\n\n  @-moz-document url-prefix() {\n    body {\n      height: 99%;\n      /* Hide scroll bar in Firefox */\n    }\n  }\n\n  td#diff-col div {\n    text-align: justify;\n    white-space: pre-wrap;\n  }\n\n  .ignored {\n    background-color: #ccc;\n    /*  border: #0d91fa 1px solid; */\n    opacity: 0.7;\n  }\n\n  .triggered {\n    background-color: #1b98f8;\n  }\n\n  /* ignored and triggered? make it obvious error */\n  .ignored.triggered {\n    background-color: #ff0000;\n  }\n\n  .tab-pane-inner#screenshot {\n    text-align: center;\n\n    img {\n      max-width: 99%;\n    }\n  }\n\n\n  // resets button margin to 0px\n  .pure-form button.reset-margin {\n    margin: 0px;\n  }\n\n  .diff-fieldset {\n    display: flex;\n    align-items: center;\n    gap: 4px;\n    flex-wrap: wrap;\n  }\n\n\n  ul#highlightSnippetActions {\n    list-style-type: none;\n    display: flex;\n    align-items: center;\n    justify-content: center;\n    gap: 1.5rem;\n    flex-wrap: wrap;\n    padding: 0;\n    margin: 0;\n\n    li {\n      display: flex;\n      flex-direction: column;\n      align-items: center;\n      text-align: center;\n      padding: 0.5rem;\n      gap: 0.3rem;\n\n      button, a {\n        white-space: nowrap;\n      }\n    }\n\n    span {\n      font-size: 0.8rem;\n      color: var(--color-text-input-description);\n    }\n  }\n\n  #cell-diff-jump-visualiser {\n    display: flex;\n    flex-direction: row;\n    gap: 1px;\n    background: var(--color-background);\n    border-radius: 3px;\n    overflow-x: hidden;\n    position: sticky;\n    top: 0;\n    z-index: 10;\n    padding-top: 1rem;\n    padding-bottom: 1rem;\n    justify-content: center;\n\n    > div {\n      flex: 1;\n      min-width: 1px;\n      max-width: 10px;\n      height: 10px;\n      background: var(--color-background-button-cancel);\n      opacity: 0.3;\n      border-radius: 1px;\n      transition: opacity 0.2s;\n      position: relative;\n\n      &.deletion {\n        background: #b30000; // Red for deletions\n        opacity: 1;\n      }\n\n      &.insertion {\n        background: #406619; // Green for insertions\n        opacity: 1;\n      }\n\n      &.note {\n        background: #406619; // Orange for changed/notes\n        opacity: 1;\n      }\n\n      &.mixed {\n        background: linear-gradient(to right, #b30000 50%, #406619 50%); // Half red, half green\n        opacity: 1;\n      }\n\n      &.current-position::after {\n        content: '';\n        position: absolute;\n        bottom: -6px;\n        left: 50%;\n        transform: translateX(-50%);\n        width: 0;\n        height: 0;\n        border-left: 4px solid transparent;\n        border-right: 4px solid transparent;\n        border-bottom: 4px solid var(--color-text);\n      }\n\n      &:hover {\n        opacity: 0.8;\n        cursor: pointer;\n      }\n    }\n  }\n}\n\n#text-diff-heading-area {\n  .snapshot-age {\n    padding: 4px;\n    margin: 0.5rem 0;\n    background-color: var(--color-background-snapshot-age);\n    border-radius: 3px;\n    font-weight: bold;\n    margin-bottom: 4px;\n\n    &.error {\n      background-color: var(--color-error-background-snapshot-age);\n      color: var(--color-error-text-snapshot-age);\n    }\n    > * {\n      padding-right: 1rem;\n    }\n  }\n}"
  },
  {
    "path": "changedetectionio/static/styles/scss/parts/_action_sidebar.scss",
    "content": "// Action Sidebar - Minimal navigation icons with light grey aesthetic\n\n.content-wrapper {\n  display: flex;\n  gap: 0;\n  width: 100%;\n  max-width: 100%;\n  position: relative;\n\n  @media only screen and (max-width: 900px) {\n    flex-direction: column;\n  }\n}\n\n.action-sidebar {\n  position: sticky;\n  top: 100px;\n  flex-shrink: 0;\n  width: 80px;\n  height: fit-content;\n  background: transparent;\n  padding: 1.5rem 0;\n  display: flex;\n  flex-direction: column;\n  gap: 0.5rem;\n  align-items: center;\n  z-index: 0;\n\n  @media only screen and (max-width: 900px) {\n    position: relative;\n    top: 0;\n    width: 100%;\n    flex-direction: row;\n    justify-content: space-around;\n    padding: 0;\n    overflow-x: auto;\n  }\n}\n\n.action-sidebar-item {\n  position: relative;\n  display: flex;\n  flex-direction: column;\n  align-items: center;\n  justify-content: center;\n  gap: 0.35rem;\n  padding: 0.75rem 0.5rem;\n  min-width: 64px;\n  text-decoration: none;\n  opacity: 0.8;\n  transition: opacity 0.2s ease;\n\n  &:hover {\n    opacity: 1;\n  }\n\n  &.active {\n    opacity: 1;\n\n    .action-icon {\n      stroke: #fff;\n      stroke-width: 2.5;\n    }\n\n    .action-label {\n      color: #fff;\n      font-weight: 700;\n    }\n  }\n}\n\n.action-icon {\n  width: 28px;\n  height: 28px;\n  stroke: #fff;\n  stroke-width: 2;\n  fill: none;\n  stroke-linecap: round;\n  stroke-linejoin: round;\n  transition: stroke 0.2s ease;\n}\n\n.action-label {\n  font-size: 0.65rem;\n  font-weight: 500;\n  text-align: center;\n  line-height: 1.1;\n  letter-spacing: 0.02em;\n  text-transform: uppercase;\n  color: #fff;\n  transition: color 0.2s ease;\n  max-width: 60px;\n  word-wrap: break-word;\n}\n\n.content-main {\n  flex: 0 1 auto;\n  width: 100%;\n  min-width: 0;\n  padding: 0;\n  display: flex;\n  flex-direction: column;\n  align-items: center;\n}\n\n// Dark mode adjustments\nhtml[data-darkmode=true] {\n  .action-icon {\n/*    stroke: #666;*/\n  }\n\n  .action-label {\n/*    color: #666;*/\n  }\n}\n"
  },
  {
    "path": "changedetectionio/static/styles/scss/parts/_arrows.scss",
    "content": ".arrow {\n  border: solid #1b98f8;\n  border-width: 0 2px 2px 0;\n  display: inline-block;\n  padding: 3px;\n\n  &.right {\n    transform: rotate(-45deg);\n    -webkit-transform: rotate(-45deg);\n  }\n\n  &.left {\n    transform: rotate(135deg);\n    -webkit-transform: rotate(135deg);\n  }\n\n  &.up, &.asc {\n    transform: rotate(-135deg);\n    -webkit-transform: rotate(-135deg);\n  }\n\n  &.down, &.desc {\n    transform: rotate(45deg);\n    -webkit-transform: rotate(45deg);\n  }\n}\n"
  },
  {
    "path": "changedetectionio/static/styles/scss/parts/_browser-steps.scss",
    "content": "\n#browser_steps {\n  /* convert rows to horizontal cells */\n  th {\n    display: none;\n  }\n\n  li {\n    &.browser-step-with-error {\n      background-color: #ffd6d6;\n      border-radius: 4px;\n    }\n    &:not(:first-child) {\n      &:hover {\n        opacity: 1.0;\n      }\n    }\n    list-style: decimal;\n    padding: 5px;\n    .control {\n      padding-left: 5px;\n      padding-right: 5px;\n      a {\n        font-size: 70%;\n      }\n    }\n    &.empty {\n      padding: 0px;\n      opacity: 0.35;\n      .control {\n        display: none;\n      }\n    }\n    &:hover {\n      background: #eee;\n    }\n    > label {\n      display: none;\n    }\n  }\n}\n\n@media only screen and (min-width: 760px) {\n\n  #browser-steps .flex-wrapper {\n    display: flex;\n    flex-flow: row;\n    height: 70vh;\n    font-size: 80%;\n\n    #browser-steps-ui {\n      flex-grow: 1; /* Allow it to grow and fill the available space */\n      flex-shrink: 1; /* Allow it to shrink if needed */\n      flex-basis: 0; /* Start with 0 base width so it stretches as much as possible */\n      background-color: #eee;\n      border-radius: 5px;\n\n    }\n  }\n\n  #browser-steps-fieldlist {\n    flex-grow: 0;      /* Don't allow it to grow */\n    flex-shrink: 0;    /* Don't allow it to shrink */\n    flex-basis: auto;  /* Base width is determined by the content */\n    max-width: 400px;  /* Set a max width to prevent overflow */\n    padding-left: 1rem;\n    overflow-y: scroll;\n  }\n\n  /*  this is duplicate :( */\n  #browsersteps-selector-wrapper {\n    height: 100% !important;\n  }\n}\n\n/*  this is duplicate :( */\n#browsersteps-selector-wrapper {\n\n  width: 100%;\n  overflow-y: scroll;\n  position: relative;\n  height: 80vh;\n\n  > img {\n    position: absolute;\n    max-width: 100%;\n  }\n\n  > canvas {\n    position: relative;\n    max-width: 100%;\n\n    &:hover {\n      cursor: pointer;\n    }\n  }\n\n  .loader {\n    position: absolute;\n    left: 50%;\n    top: 50%;\n    transform: translate(-50%, -50%);\n    z-index: 100;\n    max-width: 350px;\n    text-align: center;\n  }\n\n  /* nice tall skinny one */\n  .spinner, .spinner:after {\n    width: 80px;\n    height: 80px;\n    font-size: 3px;\n  }\n\n  #browsersteps-click-start {\n    &:hover {\n      cursor: pointer;\n    }\n    color: var(--color-grey-400);\n  }\n}"
  },
  {
    "path": "changedetectionio/static/styles/scss/parts/_conditions_table.scss",
    "content": "/* Styles for the flexbox-based table replacement for conditions */\n.fieldlist_formfields {\n  width: 100%;\n  background-color: var(--color-background, #fff);\n  border-radius: 4px;\n  border: 1px solid var(--color-border-table-cell, #cbcbcb);\n  \n  /* Header row */\n  .fieldlist-header {\n    display: flex;\n    background-color: var(--color-background-table-thead, #e0e0e0);\n    font-weight: bold;\n    border-bottom: 1px solid var(--color-border-table-cell, #cbcbcb);\n  }\n  \n  .fieldlist-header-cell {\n    flex: 1;\n    padding: 0.5em 1em;\n    text-align: left;\n    \n    &:last-child {\n      flex: 0 0 120px; /* Fixed width for actions column */\n    }\n  }\n  \n  /* Body rows */\n  .fieldlist-body {\n    display: flex;\n    flex-direction: column;\n  }\n  \n  .fieldlist-row {\n    display: flex;\n    border-bottom: 1px solid var(--color-border-table-cell, #cbcbcb);\n    \n    &:last-child {\n      border-bottom: none;\n    }\n    \n    &:nth-child(2n-1) {\n      background-color: var(--color-table-stripe, #f2f2f2);\n    }\n    \n    &.error-row {\n      background-color: var(--color-error-input, #ffdddd);\n    }\n  }\n  \n  .fieldlist-cell {\n    flex: 1;\n    padding: 0.5em 1em;\n    display: flex;\n    flex-direction: column;\n    justify-content: center;\n    \n    /* Make inputs take up full width of their cell */\n    input, select {\n      width: 100%;\n    }\n    \n    &.fieldlist-actions {\n      flex: 0 0 120px; /* Fixed width for actions column */\n      display: flex;\n      flex-direction: row;\n      align-items: center;\n      gap: 4px;\n    }\n  }\n  \n  /* Error styling */\n  ul.errors {\n    margin-top: 0.5em;\n    margin-bottom: 0;\n    padding: 0.5em;\n    background-color: var(--color-error-background-snapshot-age, #ffdddd);\n    border-radius: 4px;\n    list-style-position: inside;\n  }\n  \n  /* Responsive styles */\n  @media only screen and (max-width: 760px) {\n    .fieldlist-header, .fieldlist-row {\n      flex-direction: column;\n    }\n    \n    .fieldlist-header-cell {\n      display: none;\n    }\n    \n    .fieldlist-row {\n      padding: 0.5em 0;\n      border-bottom: 2px solid var(--color-border-table-cell, #cbcbcb);\n    }\n    \n    .fieldlist-cell {\n      padding: 0.25em 0.5em;\n      \n      &.fieldlist-actions {\n        flex: 1;\n        justify-content: flex-start;\n        padding-top: 0.5em;\n      }\n    }\n    \n    /* Add some spacing between fields on mobile */\n    .fieldlist-cell:not(:last-child) {\n      margin-bottom: 0.5em;\n    }\n    \n    /* Label each cell on mobile view */\n    .fieldlist-cell::before {\n      content: attr(data-label);\n      font-weight: bold;\n      margin-bottom: 0.25em;\n    }\n  }\n}\n\n/* Button styling */\n.fieldlist_formfields {\n  .addRuleRow, .removeRuleRow, .verifyRuleRow {\n    cursor: pointer;\n    border: none;\n    padding: 4px 8px;\n    border-radius: 3px;\n    font-weight: bold;\n    background-color: #aaa;\n    color: var(--color-foreground-text, #fff);\n\n    &:hover {\n      background-color: #999;\n    }\n  }\n\n}"
  },
  {
    "path": "changedetectionio/static/styles/scss/parts/_darkmode.scss",
    "content": "\n.toggle-light-mode {\n  /* default */\n  .icon-dark {\n    display: none;\n  }\n}\n\nhtml[data-darkmode=\"true\"] {\n  .toggle-light-mode {\n    .icon-light {\n      display: none;\n    }\n\n    .icon-dark {\n      display: block;\n    }\n  }\n}\n\n\n\n\n"
  },
  {
    "path": "changedetectionio/static/styles/scss/parts/_diff_image.scss",
    "content": "body.processor-image_ssim_diff {\n  #edit-text-filter {\n    .text-filtering {\n      display: none;\n    }\n  }\n  #conditions-tab {\n    display: none;\n  }\n}"
  },
  {
    "path": "changedetectionio/static/styles/scss/parts/_edit.scss",
    "content": "ul#conditions_match_logic {\n    list-style: none;\n  input, label, li {\n    display: inline-block;\n  }\n  li {\n    padding-right: 1em;\n  }\n}\n"
  },
  {
    "path": "changedetectionio/static/styles/scss/parts/_extra_browsers.scss",
    "content": "ul#requests-extra_browsers {\n  list-style: none;\n  /* tidy up the table to look more \"inline\" */\n  li {\n    > label {\n      display: none;\n    }\n\n  }\n\n  /* each proxy entry is a `table` */\n  table {\n    tr {\n      display: table-row; // default display for small screens\n      input[type=text] {\n        width: 100%;\n      }\n    }\n  }\n  \n  // apply inline display for larger screens\n  @media only screen and (min-width: 1280px) {\n    table {\n      tr {\n        display: inline;\n        input[type=text] {\n          width: 100%;\n        }\n      }\n    }\n  }\n}\n\n#extra-browsers-setting {\n  border: 1px solid var(--color-grey-800);\n  border-radius: 4px;\n  margin: 1em;\n   padding: 1em;\n}"
  },
  {
    "path": "changedetectionio/static/styles/scss/parts/_extra_proxies.scss",
    "content": "ul#requests-extra_proxies {\n  list-style: none;\n  /* tidy up the table to look more \"inline\" */\n  li {\n    > label {\n      display: none;\n    }\n\n  }\n\n  /* each proxy entry is a `table` */\n  table {\n    tr {\n      display: table-row; // default display for small screens\n      input[type=text] {\n        width: 100%;\n      }\n    }\n  }\n  \n  // apply inline display for large screens\n  @media only screen and (min-width: 1024px) {\n    table {\n      tr {\n        display: inline;\n      }\n    }\n  }\n}\n\n#request {\n  /* Auto proxy scan/checker */\n  label[for=proxy] {\n    display: inline-block;\n  }\n}\n\nbody.proxy-check-active {\n  #request {\n    // Padding set by flex layout\n    /*\n    .proxy-status {\n      width: 2em;\n    }\n    */\n\n    .proxy-check-details {\n      font-size: 80%;\n      color: #555;\n      display: block;\n      padding-left: 2em;\n      max-width: 500px;\n    }\n\n    .proxy-timing {\n      font-size: 80%;\n      padding-left: 1rem;\n      color: var(--color-link);\n    }\n  }\n}\n\n\n#recommended-proxy {\n  display: grid;\n  gap: 2rem;\n  padding-bottom: 1em;\n  \n  @media  (min-width: 991px) {\n    grid-template-columns: repeat(2, 1fr);\n  }\n\n  > div {\n    border: 1px #aaa solid;\n    border-radius: 4px;\n    padding: 1em;\n  }\n}\n\n#extra-proxies-setting {\n  border: 1px solid var(--color-grey-800);\n  border-radius: 4px;\n    margin: 1em;\n   padding: 1em;\n}\n"
  },
  {
    "path": "changedetectionio/static/styles/scss/parts/_hamburger_menu.scss",
    "content": "// Hamburger Menu for Mobile Navigation\n@use \"../settings\" as *;\n\n.hamburger-menu {\n  display: none;\n  background: transparent;\n  border: none;\n  cursor: pointer;\n  padding: 0.5rem;\n  z-index: 10001;\n  position: relative;\n\n  @media only screen and (max-width: $desktop-wide-breakpoint) {\n    display: flex;\n    flex-direction: column;\n    justify-content: center;\n    align-items: center;\n  }\n}\n\n.hamburger-icon {\n  width: 24px;\n  height: 20px;\n  position: relative;\n  display: flex;\n  flex-direction: column;\n  justify-content: space-between;\n\n  span {\n    display: block;\n    height: 3px;\n    width: 100%;\n    background: var(--color-text);\n    border-radius: 2px;\n    transition: all 0.3s cubic-bezier(0.68, -0.55, 0.265, 1.55);\n    transform-origin: center;\n  }\n}\n\n.hamburger-menu.active {\n  .hamburger-icon span:nth-child(1) {\n    transform: translateY(8.5px) rotate(45deg);\n  }\n\n  .hamburger-icon span:nth-child(2) {\n    opacity: 0;\n    transform: translateX(-10px);\n  }\n\n  .hamburger-icon span:nth-child(3) {\n    transform: translateY(-8.5px) rotate(-45deg);\n  }\n}\n\n// Mobile menu overlay\n.mobile-menu-overlay {\n  display: none;\n  position: fixed;\n  top: 0;\n  left: 0;\n  right: 0;\n  bottom: 0;\n  background: rgba(0, 0, 0, 0.5);\n  z-index: 9999;\n  opacity: 0;\n  transition: opacity 0.3s ease;\n\n  &.active {\n    display: block;\n    opacity: 1;\n  }\n}\n\n// Mobile menu drawer\n.mobile-menu-drawer {\n  position: fixed;\n  top: 0;\n  right: -280px;\n  width: 280px;\n  height: 100%;\n  background: var(--color-background);\n  opacity: 1;\n  box-shadow: -2px 0 8px rgba(0, 0, 0, 0.15);\n  z-index: 10000;\n  transition: right 0.3s cubic-bezier(0.68, -0.55, 0.265, 1.55);\n  overflow-y: auto;\n  padding-top: 60px;\n\n  &.active {\n    right: 0;\n  }\n\n  .mobile-menu-items {\n    list-style: none;\n    padding: 1rem 0;\n    margin: 0;\n\n    li {\n      border-bottom: 1px solid var(--color-border-table-cell);\n\n      >* {\n        display: block;\n        padding: 1rem 1.5rem;\n        color: var(--color-text);\n        text-decoration: none;\n        font-weight: 500;\n        transition: background 0.2s ease;\n\n        &:hover {\n          background: var(--color-background-menu-link-hover);\n        }\n      }\n      &#menu-pause, &#menu-mute {\n        display: none;\n      }\n    }\n  }\n}\n\n// Logo styling\n.logo-cdio {\n  font-weight: bold;\n  font-size: 1.1rem;\n\n  .logo-cd {\n    color: var(--color-grey-500);\n  }\n\n  .logo-io {\n    color: var(--color-text);\n  }\n}\n\n// Always visible items container\n.menu-always-visible {\n  display: flex;\n  align-items: center;\n  gap: 0.5rem;\n  margin-left: auto;\n}\n\n// Hide regular menu items on mobile (but not in mobile drawer)\n@media only screen and (max-width: $desktop-wide-breakpoint) {\n  #top-right-menu .menu-collapsible {\n    display: none !important;\n  }\n\n  .pure-menu-horizontal {\n    overflow-x: visible !important;\n  }\n\n  #nav-menu {\n    overflow-x: visible !important;\n  }\n}\n\n// Desktop - hide mobile menu elements\n@media only screen and (min-width: 1025px) {\n  .hamburger-menu,\n  .mobile-menu-drawer,\n  .mobile-menu-overlay {\n    display: none !important;\n  }\n}\n\nhtml[data-darkmode=true] {\n  .mobile-menu-drawer {\n    box-shadow: -2px 0 8px rgba(0, 0, 0, 0.4);\n  }\n}\n"
  },
  {
    "path": "changedetectionio/static/styles/scss/parts/_language.scss",
    "content": "#language-selector-flag {\n  display: inline-block;\n  width: 1.2em;\n  height: 1.2em;\n  vertical-align: middle;\n  border-radius: 50%;\n  overflow: hidden;\n  opacity: 0.6;\n  &:hover {\n    opacity: 1.0;\n  }\n}\n\n\n// Language Selector Modal Styles\n.language-list {\n  display: flex;\n  flex-direction: column;\n  gap: 0.5rem;\n  padding: 0.5rem 0;\n}\n\n.language-option {\n  display: flex;\n  align-items: center;\n  gap: 1rem;\n  padding: 0.25rem;\n  border-radius: 4px;\n  transition: background-color 0.2s ease;\n  text-decoration: none;\n  color: var(--color-text);\n  border: 1px solid transparent;\n\n  &:hover {\n    background-color: var(--color-background-menu-link-hover);\n    border-color: var(--color-border-table-cell);\n  }\n\n  &.active {\n    background-color: var(--color-link);\n    color: var(--color-text-button);\n    font-weight: 600;\n  }\n\n  .flag {\n    font-size: 1.5rem;\n    flex-shrink: 0;\n  }\n\n  .language-name {\n    flex-grow: 1;\n    font-size: 1rem;\n  }\n}\n\n#language-modal {\n  .language-list {\n    .lang-option {\n      display: inline-block;\n      width: 1.5em;\n      height: 1.5em;\n      vertical-align: middle;\n      margin-right: 0.5em;\n      border-radius: 50%;\n      overflow: hidden;\n    }\n  }\n}\n\n"
  },
  {
    "path": "changedetectionio/static/styles/scss/parts/_lister_extra.scss",
    "content": ".watch-table {\n  &.favicon-not-enabled {\n    tr {\n      .favicon {\n        display: none;\n      }\n    }\n  }\n\n  tr {\n    /* make the icons and the text inline-ish */\n    td.inline.title-col {\n      .flex-wrapper {\n        display: flex;\n        align-items: center;\n        gap: 4px;\n      }\n    }\n  }\n\n\n  td,\n  th {\n    vertical-align: middle;\n  }\n\n  tr.has-favicon {\n    &.unviewed {\n      img.favicon {\n        opacity: 1.0 !important;\n      }\n    }\n  }\n\n  .status-icons {\n    white-space: nowrap;\n    display: flex;\n    align-items: center; /* Vertical centering */\n    gap: 4px; /* Space between image and text */\n    > * {\n      vertical-align: middle;\n    }\n  }\n}\n\n.title-col {\n  /* Optional, for spacing */\n  padding: 10px;\n}\n\n.title-wrapper {\n  display: flex;\n  align-items: center; /* Vertical centering */\n  gap: 10px; /* Space between image and text */\n}\n\n/* Make sure .title-col-inner doesn't collapse or misalign */\n.title-col-inner {\n  display: inline-block;\n  vertical-align: middle;\n}\n\n/* favicon styling */\n.watch-table {\n  img.favicon {\n    vertical-align: middle;\n    max-width: 25px;\n    max-height: 25px;\n    height: 25px;\n    padding-right: 4px;\n  }\n\n    // Reserved for future use\n  /*  &.thumbnail-type-screenshot {\n      tr.has-favicon {\n        td.inline.title-col {\n          img.thumbnail {\n            background-color: #fff; !* fallback bg for SVGs without bg *!\n            border-radius: 4px; !* subtle rounded corners *!\n            border: 1px solid #ddd; !* light border for contrast *!\n            box-shadow: 0 2px 6px rgba(0, 0, 0, 0.15); !* soft shadow *!\n            filter: contrast(1.05) saturate(1.1) drop-shadow(0 0 0.5px rgba(0, 0, 0, 0.2));\n            object-fit: cover; !* crop/fill if needed *!\n            opacity: 0.8;\n            max-width: 30px;\n            max-height: 30px;\n            height: 30px;\n          }\n        }\n      }\n    }*/\n}"
  },
  {
    "path": "changedetectionio/static/styles/scss/parts/_login_form.scss",
    "content": "// Modern Login Form - Friendly and Welcoming Design\n\n.login-form {\n  min-height: 52vh;\n  display: flex;\n  align-items: center;\n  justify-content: center;\n  padding: 2rem 1rem;\n\n  .inner {\n    background: var(--color-background);\n    border-radius: 16px;\n    box-shadow:\n      0 10px 40px rgba(0, 0, 0, 0.08),\n      0 2px 8px rgba(0, 0, 0, 0.04);\n    padding: 3rem 2.5rem;\n    width: 100%;\n    max-width: 420px;\n    position: relative;\n    overflow: hidden;\n    transition: transform 0.3s ease, box-shadow 0.3s ease;\n\n    &:hover {\n\n      box-shadow:\n        0 15px 50px rgba(0, 0, 0, 0.12),\n        0 5px 15px rgba(0, 0, 0, 0.06);\n    }\n  }\n\n  form {\n    margin: 0;\n  }\n\n  fieldset {\n    border: none;\n    padding: 0;\n    margin: 0;\n  }\n\n  .pure-control-group {\n    margin-bottom: 1.75rem;\n\n    &:last-of-type {\n      margin-bottom: 0;\n      margin-top: 2rem;\n    }\n  }\n\n  label {\n    display: block;\n    margin-bottom: 0.5rem;\n    font-weight: 600;\n    font-size: 0.9rem;\n    color: var(--color-text);\n    letter-spacing: 0.01em;\n  }\n\n  input[type=\"password\"] {\n    width: 100%;\n    padding: 0.875rem 1rem;\n    border: 2px solid var(--color-grey-800);\n    border-radius: 8px;\n    font-size: 1rem;\n    background: var(--color-background-input);\n    color: var(--color-text-input);\n    transition: all 0.2s ease;\n    box-sizing: border-box;\n\n    &:focus {\n      outline: none;\n      border-color: var(--color-link);\n      box-shadow: 0 0 0 3px rgba(27, 152, 248, 0.1);\n      transform: translateY(-1px);\n    }\n\n    &::placeholder {\n      color: var(--color-text-input-placeholder);\n    }\n  }\n\n  button[type=\"submit\"] {\n    width: 100%;\n    padding: 0.875rem 1.5rem;\n    font-size: 1rem;\n    font-weight: 600;\n    border-radius: 8px;\n    border: none;\n    background: var(--color-background-button-primary);\n    color: var(--color-text-button);\n    cursor: pointer;\n    transition: all 0.2s ease;\n    box-shadow: 0 2px 8px rgba(27, 152, 248, 0.2);\n\n    &:hover {\n      box-shadow: 0 4px 12px rgba(27, 152, 248, 0.3);\n      background: #0066cc;\n    }\n\n    &:active {\n      transform: translateY(0);\n      box-shadow: 0 2px 4px rgba(27, 152, 248, 0.2);\n    }\n  }\n}\n\n// Messages styling for login page\n.content-main > ul.messages {\n  position: fixed;\n  top: 120px;\n  left: 50%;\n  transform: translateX(-50%);\n  list-style: none;\n  padding: 0;\n  margin: 0;\n  z-index: 1000;\n  min-width: 300px;\n  max-width: 500px;\n\n  li {\n    padding: 1rem 1.25rem;\n    border-radius: 8px;\n    font-size: 0.95rem;\n    line-height: 1.5;\n    font-weight: 500;\n    box-shadow: 0 4px 12px rgba(0, 0, 0, 0.15);\n    animation: slideDown 0.3s ease-out;\n    border: 2px solid transparent;\n\n    &.error {\n      background: #fee;\n      border: 2px solid #ef4444;\n      color: #991b1b;\n      font-weight: 600;\n    }\n\n    &.success {\n      background: #f0fdf4;\n      border: 2px solid #10b981;\n      color: #166534;\n    }\n\n    &.info,\n    &.message {\n      background: #eff6ff;\n      border: 2px solid #3b82f6;\n      color: #1e40af;\n    }\n  }\n}\n\n@keyframes slideDown {\n  from {\n    opacity: 0;\n    transform: translateY(-20px);\n  }\n  to {\n    opacity: 1;\n    transform: translateY(0);\n  }\n}\n\n// Dark mode adjustments\nhtml[data-darkmode=\"true\"] {\n  .login-form {\n    .inner {\n      box-shadow:\n        0 10px 40px rgba(0, 0, 0, 0.4),\n        0 2px 8px rgba(0, 0, 0, 0.2);\n\n      &:hover {\n        box-shadow:\n          0 15px 50px rgba(0, 0, 0, 0.5),\n          0 5px 15px rgba(0, 0, 0, 0.3);\n      }\n    }\n\n    input[type=\"password\"] {\n      border-color: var(--color-grey-400);\n\n      &:focus {\n        border-color: var(--color-link);\n      }\n    }\n  }\n\n  .content-main > ul.messages {\n    li {\n      box-shadow: 0 4px 12px rgba(0, 0, 0, 0.4);\n\n      &.error {\n        background: #4a1d1d;\n        border-color: #ef4444;\n        color: #fca5a5;\n      }\n\n      &.success {\n        background: #1a3a2a;\n        border-color: #10b981;\n        color: #86efac;\n      }\n\n      &.info,\n      &.message {\n        background: #1e3a5f;\n        border-color: #3b82f6;\n        color: #93c5fd;\n      }\n    }\n  }\n}\n\n// Mobile adjustments\n@media only screen and (max-width: 768px) {\n  .login-form {\n    min-height: auto;\n    padding: 1rem 0.5rem;\n    padding-top: 5rem; // Space for error message\n\n    .inner {\n      padding: 2rem 1.5rem;\n      border-radius: 12px;\n    }\n  }\n\n  .content-main > ul.messages {\n    top: 70px; // Higher up on mobile to avoid overlap\n    left: 10px;\n    right: 10px;\n    transform: none;\n    min-width: auto;\n  }\n}\n"
  },
  {
    "path": "changedetectionio/static/styles/scss/parts/_love.scss",
    "content": "#overlay {\n\n  opacity: 0.95;\n  position: fixed;\n\n  width: 350px;\n  max-width: 100%;\n  height: 100%;\n  top: 0;\n  right: -350px;\n  background-color: var(--color-table-stripe);\n  z-index: 2;\n\n  transform: translateX(0);\n  transition: transform .5s ease;\n\n\n  &.visible {\n    transform: translateX(-100%);\n\n  }\n\n  .content {\n    font-size: 0.875rem;\n    padding: 1rem;\n    margin-top: 5rem;\n    max-width: 400px;\n    color: var(--color-watch-table-row-text);\n  }\n}\n\n#heartpath {\n  &:hover {\n    fill: #ff0000 !important;\n    transition: all ease 0.3s !important;\n  }\n  transition: all ease 0.3s !important;\n}\n"
  },
  {
    "path": "changedetectionio/static/styles/scss/parts/_menu.scss",
    "content": ".pure-menu-link {\n  padding: 0.5rem 1em;\n  line-height: 1.2rem;\n}\n#menu-mute, #menu-pause {\n  padding-left: 0.3rem;\n  padding-right: 0.3rem;\n  img {\n    height: 1.2rem;\n  }\n}\n\n.pure-menu-item {\n  svg {\n    height: 1.2rem;\n  }\n  * {\n    vertical-align: middle;\n  }\n  .github-link {\n    height: 1.8rem;\n    display: block;\n    svg {\n      height: 100%;\n    }\n  }\n  .bi-heart {\n    &:hover {\n      cursor: pointer;\n    }\n  }\n\n  // Active menu item styling\n  &.active {\n    .pure-menu-link {\n      background-color: var(--color-background-menu-link-hover);\n      color: var(--color-text-menu-link-hover);\n    }\n  }\n}\n\n#cdio-logo {\n  padding-left: 0.5em;\n}\n\n#inline-menu-extras-group {\n  >* {\n    display: inline-block;\n  }\n}"
  },
  {
    "path": "changedetectionio/static/styles/scss/parts/_minitabs.scss",
    "content": ".minitabs-wrapper {\n  width: 100%;\n\n  > div[id] {\n    padding: 20px;\n    border: 1px solid #ccc;\n    border-top: none;\n  }\n\n  .minitabs-content {\n    width: 100%;\n    display: flex;\n    > div {\n      flex: 1 1 auto;\n      min-width: 0;\n      overflow: scroll;\n    }\n  }\n\n  .minitabs {\n    display: flex;\n    border-bottom: 1px solid #ccc;\n  }\n\n  .minitab {\n    flex: 1;\n    text-align: center;\n    padding: 12px 0;\n    text-decoration: none;\n    color: #333;\n    background-color: #f1f1f1;\n    border: 1px solid #ccc;\n    border-bottom: none;\n    cursor: pointer;\n    transition: background-color 0.3s;\n  }\n\n  .minitab:hover {\n    background-color: #ddd;\n  }\n\n  .minitab.active {\n    background-color: #fff;\n    font-weight: bold;\n  }\n\n}\n"
  },
  {
    "path": "changedetectionio/static/styles/scss/parts/_modal.scss",
    "content": "/**\n * Modal dialog styles using HTML5  element\n * Provides modern, accessible confirmation dialogs\n */\n\n.modal-dialog {\n  border: none;\n  border-radius: 10px;\n  padding: 0;\n  background: var(--color-background);\n  color: var(--color-text);\n  box-shadow: 0 5px 20px rgba(0, 0, 0, 0.3);\n  max-width: 500px;\n  width: 90%;\n\n  &::backdrop {\n    background: rgba(0, 0, 0, 0.6);\n    backdrop-filter: blur(3px);\n    animation: fadeIn 0.2s ease-out;\n  }\n\n  &[open] {\n    animation: slideIn 0.25s ease-out;\n  }\n\n  .modal-header {\n    padding: 1.5rem;\n    border-bottom: 1px solid var(--color-border-table-cell);\n    display: flex;\n    align-items: center;\n    gap: 1rem;\n\n    .modal-icon {\n      font-size: 2rem;\n      line-height: 1;\n      flex-shrink: 0;\n\n      &.warning {\n        color: var(--color-warning);\n      }\n\n      &.danger {\n        color: var(--color-background-button-error);\n      }\n\n      &.info {\n        color: var(--color-background-button-primary);\n      }\n    }\n\n    .modal-title {\n      font-size: 1.3rem;\n      font-weight: bold;\n      margin: 0;\n      color: var(--color-text);\n    }\n  }\n\n  .modal-body {\n    padding: 1.5rem;\n    line-height: 1.6;\n\n    p {\n      margin: 0 0 1rem 0;\n\n      &:last-child {\n        margin-bottom: 0;\n      }\n    }\n\n    strong {\n      color: var(--color-text);\n      font-weight: 600;\n    }\n  }\n\n  .modal-footer {\n    padding: 1rem 1.5rem;\n    border-top: 1px solid var(--color-border-table-cell);\n    display: flex;\n    gap: 0.75rem;\n    justify-content: flex-end;\n    background: var(--color-grey-900);\n\n    button {\n      padding: 0.6rem 1.5rem;\n      border: none;\n      border-radius: 4px;\n      cursor: pointer;\n      font-weight: 500;\n      transition: all 0.2s ease;\n      font-size: 0.95rem;\n\n      &:hover {\n        transform: translateY(-1px);\n        box-shadow: 0 2px 8px rgba(0, 0, 0, 0.15);\n      }\n\n      &:active {\n        transform: translateY(0);\n      }\n\n      &.modal-btn-cancel {\n        background: var(--color-background-button-cancel);\n        color: var(--color-grey-200);\n\n        &:hover {\n          background: var(--color-grey-700);\n        }\n      }\n\n      &.modal-btn-confirm {\n        background: var(--color-background-button-primary);\n        color: var(--color-white);\n\n        &:hover {\n          opacity: 0.9;\n        }\n      }\n\n      &.modal-btn-danger {\n        background: var(--color-background-button-error);\n        color: var(--color-white);\n\n        &:hover {\n          background: var(--color-dark-red);\n        }\n      }\n\n      &.modal-btn-warning {\n        background: var(--color-background-button-warning);\n        color: var(--color-white);\n\n        &:hover {\n          opacity: 0.9;\n        }\n      }\n    }\n  }\n}\n\n// Dark mode adjustments\nhtml[data-darkmode=\"true\"] {\n  .modal-dialog {\n    box-shadow: 0 5px 30px rgba(0, 0, 0, 0.7);\n\n    .modal-footer {\n      background: var(--color-grey-200);\n    }\n  }\n}\n\n// Animations\n@keyframes fadeIn {\n  from {\n    opacity: 0;\n  }\n  to {\n    opacity: 1;\n  }\n}\n\n@keyframes slideIn {\n  from {\n    opacity: 0;\n    transform: translateY(-20px) scale(0.95);\n  }\n  to {\n    opacity: 1;\n    transform: translateY(0) scale(1);\n  }\n}\n\n// Mobile responsive\n@media only screen and (max-width: 760px) {\n  .modal-dialog {\n    width: 95%;\n    max-width: none;\n\n    .modal-header {\n      padding: 1rem;\n\n      .modal-title {\n        font-size: 1.1rem;\n      }\n    }\n\n    .modal-body {\n      padding: 1rem;\n      font-size: 0.95rem;\n    }\n\n    .modal-footer {\n      padding: 0.75rem 1rem;\n      flex-wrap: wrap;\n\n      button {\n        flex: 1;\n        min-width: 120px;\n      }\n    }\n  }\n}\n"
  },
  {
    "path": "changedetectionio/static/styles/scss/parts/_notification_bubble.scss",
    "content": "// Reusable notification bubble for action sidebar icons\n\n.action-sidebar-item {\n  position: relative;\n\n  .notification-bubble {\n    position: absolute;\n    top: 8px;\n    left: 8px;\n    min-width: 18px;\n    height: 18px;\n    background: #ff4444;\n    color: #fff;\n    font-size: 10px;\n    font-weight: 700;\n    line-height: 18px;\n    text-align: center;\n    border-radius: 9px;\n    padding: 0 2px;\n    box-shadow: 0 2px 4px rgba(0, 0, 0, 0.3);\n    pointer-events: none;\n    transition: all 0.2s ease;\n    display: none;\n\n    // Red bubble for errors/urgent\n    &.red-bubble {\n      background: #ff4444;\n    }\n\n    // Blue bubble for informational\n    &.blue-bubble {\n      background: #4a9eff;\n      color: #fff;\n    }\n\n    &.visible {\n      display: block;\n    }\n\n    // Pulse animation when value changes\n    &.pulse {\n      animation: bubblePulse 0.4s ease-out;\n    }\n\n    // Large numbers get smaller font\n    &.large-number {\n      font-size: 8px;\n      min-width: 20px;\n      height: 20px;\n      line-height: 20px;\n      border-radius: 10px;\n    }\n  }\n}\n\n@keyframes bubblePulse {\n  0% {\n    transform: scale(1);\n  }\n  50% {\n    transform: scale(1.3);\n  }\n  100% {\n    transform: scale(1);\n  }\n}\n\n// Dark mode adjustments\nhtml[data-darkmode=true] {\n  .notification-bubble {\n    box-shadow: 0 2px 6px rgba(0, 0, 0, 0.6);\n  }\n}\n"
  },
  {
    "path": "changedetectionio/static/styles/scss/parts/_pagination.scss",
    "content": ".pagination-page-info {\n  text-transform: capitalize;\n}\n\n.pagination.menu {\n  > * {\n    display: inline-block;\n  }\n\n  li {\n    display: inline-block;\n  }\n\n  a {\n    padding: 0.65rem;\n    margin: 3px;\n    border: none;\n    background: #444;\n    border-radius: 2px;\n    color: var(--color-text-button);\n    &.disabled {\n      display: none;\n    }\n    &.active {\n      font-weight: bold;\n      background: #888;\n    }\n\n    &:hover {\n      background: #999;\n    }\n  }\n\n\n}"
  },
  {
    "path": "changedetectionio/static/styles/scss/parts/_preview_text_filter.scss",
    "content": "@use \"minitabs\";\n\nbody.preview-text-enabled {\n\n  @media (min-width: 800px) {\n    #filters-and-triggers > div {\n      display: flex; /* Establishes Flexbox layout */\n      gap: 20px; /* Adds space between the columns */\n      position: relative; /* Ensures the sticky positioning is relative to this parent */\n    }\n  }\n\n  /* layout of the page */\n  #edit-text-filter, #text-preview {\n    flex: 1; /* Each column takes an equal amount of available space */\n    align-self: flex-start; /* Aligns the right column to the start, allowing it to maintain its content height */\n  }\n\n  #edit-text-filter {\n    #pro-tips {\n      display: none;\n    }\n  }\n\n  #text-preview {\n    position: sticky;\n    top: 20px;\n    padding-top: 1rem;\n    padding-bottom: 1rem;\n    display: block !important;\n  }\n\n  #activate-text-preview {\n      background-color: var(--color-grey-500);\n  }\n\n  /* actual preview area */\n  .monospace-preview {\n    background: var(--color-background-input);\n    border: 1px solid var(--color-grey-600);\n    padding: 1rem;\n    color: var(--color-text-input);\n    font-family: \"Courier New\", Courier, monospace; /* Sets the font to a monospace type */\n    font-size: 70%;\n    word-break: break-word;\n    white-space: pre-wrap; /* Preserves whitespace and line breaks like 
 */\n  }\n}\n\n#activate-text-preview {\n  right: 0;\n  position: absolute;\n  z-index: 3;\n  box-shadow: 1px 1px 4px var(--color-shadow-jump);\n}\n"
  },
  {
    "path": "changedetectionio/static/styles/scss/parts/_search_modal.scss",
    "content": "// Search Modal Styles\n\n#search-modal {\n  .modal-body {\n    padding: 2rem 1.5rem;\n\n    .pure-control-group {\n      padding-bottom: 0;\n\n      label {\n        display: block;\n        margin-bottom: 0.5rem;\n        font-size: 0.9rem;\n        font-weight: 600;\n        color: var(--color-text);\n      }\n\n      #search-modal-input {\n        width: 100%;\n        max-width: 100%;\n        box-sizing: border-box;\n        padding: 0.6rem 0.8rem;\n        font-size: 1rem;\n        border: 1px solid var(--color-border-input);\n        border-radius: 4px;\n        background-color: var(--color-background-input);\n        color: var(--color-text-input);\n        box-shadow: inset 0 1px 3px var(--color-shadow-input);\n        transition: border-color 0.2s ease, box-shadow 0.2s ease;\n\n        &:focus {\n          outline: none;\n          border-color: var(--color-link);\n          box-shadow: 0 0 0 3px rgba(27, 152, 248, 0.1);\n        }\n\n        &::placeholder {\n          color: var(--color-text-input-placeholder);\n          opacity: 0.7;\n        }\n      }\n    }\n  }\n}\n\n// Dark mode adjustments\nhtml[data-darkmode=true] {\n  #search-modal {\n    #search-modal-input {\n      &:focus {\n        box-shadow: 0 0 0 3px rgba(89, 189, 251, 0.15);\n      }\n    }\n  }\n}\n"
  },
  {
    "path": "changedetectionio/static/styles/scss/parts/_socket.scss",
    "content": "// Styles for Socket.IO real-time updates\nbody.checking-now {\n  #checking-now-fixed-tab {\n    display: block !important;\n  }\n}\n\n#checking-now-fixed-tab {\n  background: #ccc;\n  border-radius: 5px;\n  bottom: 0;\n  color: var(--color-text);\n  display: none;\n  font-size: 0.8rem;\n  left: 0;\n  padding: 5px;\n  position: fixed;\n}\n\n\n\n\n"
  },
  {
    "path": "changedetectionio/static/styles/scss/parts/_spinners.scss",
    "content": "\n/* spinner */\n.spinner,\n.spinner:after {\n  border-radius: 50%;\n  width: 10px;\n  height: 10px;\n}\n.spinner {\n  margin: 0px auto;\n  font-size: 3px;\n  vertical-align: middle;\n  display: inline-block;\n  text-indent: -9999em;\n  border-top: 1.1em solid rgba(38,104,237, 0.2);\n  border-right: 1.1em solid rgba(38,104,237, 0.2);\n  border-bottom: 1.1em solid rgba(38,104,237, 0.2);\n  border-left: 1.1em solid #2668ed;\n  -webkit-transform: translateZ(0);\n  -ms-transform: translateZ(0);\n  transform: translateZ(0);\n  -webkit-animation: load8 1.1s infinite linear;\n  animation: load8 1.1s infinite linear;\n}\n@-webkit-keyframes load8 {\n  0% {\n    -webkit-transform: rotate(0deg);\n    transform: rotate(0deg);\n  }\n  100% {\n    -webkit-transform: rotate(360deg);\n    transform: rotate(360deg);\n  }\n}\n@keyframes load8 {\n  0% {\n    -webkit-transform: rotate(0deg);\n    transform: rotate(0deg);\n  }\n  100% {\n    -webkit-transform: rotate(360deg);\n    transform: rotate(360deg);\n  }\n}"
  },
  {
    "path": "changedetectionio/static/styles/scss/parts/_tabs.scss",
    "content": "body.wrapped-tabs {\n  .tabs {\n    ul {\n      grid-template-columns: repeat(auto-fill, minmax(var(--tab-width, 180px), 1fr));\n      grid-auto-flow: row;\n      grid-auto-columns: unset;\n      gap: 0;\n      column-gap: 5px;\n    }\n\n    ul li {\n      border-radius: 0;\n    }\n  }\n}\n\n.tabs {\n  ul {\n    margin: 0px;\n    padding: 0px;\n    display: grid;\n    grid-auto-flow: column;\n    grid-auto-columns: max-content;\n    gap: 5px;\n    list-style: none;\n\n    li {\n      white-space: nowrap;\n      color: var(--color-text-tab);\n      border-top-left-radius: 5px;\n      border-top-right-radius: 5px;\n      background-color: var(--color-background-tab);\n\n      &:not(.active) {\n        &:hover {\n          background-color: var(--color-background-tab-hover);\n        }\n      }\n\n      &.active,\n      :target {\n        background-color: var(--color-background);\n\n        a {\n          color: var(--color-text-tab-active);\n          font-weight: bold;\n        }\n      }\n\n      a {\n        display: block;\n        padding: 0.7em;\n        color: var(--color-text-tab);\n      }\n    }\n  }\n}\n"
  },
  {
    "path": "changedetectionio/static/styles/scss/parts/_toast.scss",
    "content": "// Toast Notification System\n// Modern, animated toast notifications\n\n.toast-container {\n  position: fixed;\n  display: flex;\n  flex-direction: column;\n  gap: 0.75rem;\n  pointer-events: none;\n  z-index: 10000;\n\n  // Positioning\n  &.toast-top-right {\n    top: 20px;\n    right: 20px;\n  }\n\n  &.toast-top-center {\n    top: 100px;\n    left: 50%;\n    transform: translateX(-50%);\n  }\n\n  &.toast-top-left {\n    top: 20px;\n    left: 20px;\n  }\n\n  &.toast-bottom-right {\n    bottom: 20px;\n    right: 20px;\n  }\n\n  &.toast-bottom-center {\n    bottom: 20px;\n    left: 50%;\n    transform: translateX(-50%);\n  }\n\n  &.toast-bottom-left {\n    bottom: 20px;\n    left: 20px;\n  }\n}\n\n.toast {\n  position: relative;\n  display: flex;\n  align-items: center;\n  gap: 0.75rem;\n  min-width: 300px;\n  max-width: 500px;\n  padding: 1rem 1.25rem;\n  background: var(--color-background);\n  border-radius: 8px;\n  box-shadow: 0 4px 12px rgba(0, 0, 0, 0.15), 0 0 0 1px rgba(0, 0, 0, 0.05);\n  pointer-events: auto;\n  overflow: hidden;\n  opacity: 0;\n  transform: translateY(-50px);\n  transition: all 0.3s cubic-bezier(0.68, -0.55, 0.265, 1.55);\n  font-family: inherit;\n\n  &.toast-show {\n    opacity: 1;\n    transform: translateY(0);\n  }\n\n  &.toast-hide {\n    opacity: 0;\n    transform: translateY(-50px) scale(0.95);\n  }\n\n  // Toast types\n  &.toast-success {\n    border-left: 4px solid #10b981;\n\n    .toast-icon {\n      color: #10b981;\n    }\n  }\n\n  &.toast-error {\n    border-left: 4px solid #ef4444;\n\n    .toast-icon {\n      color: #ef4444;\n    }\n  }\n\n  &.toast-warning {\n    border-left: 4px solid #f59e0b;\n\n    .toast-icon {\n      color: #f59e0b;\n    }\n  }\n\n  &.toast-info {\n    border-left: 4px solid #3b82f6;\n\n    .toast-icon {\n      color: #3b82f6;\n    }\n  }\n\n  &.toast-default {\n    border-left: 4px solid var(--color-grey-500);\n  }\n}\n\n.toast-icon {\n  flex-shrink: 0;\n  width: 24px;\n  height: 24px;\n\n  svg {\n    width: 100%;\n    height: 100%;\n  }\n}\n\n.toast-message {\n  flex: 1;\n  font-size: 0.875rem;\n  line-height: 1.5;\n  color: var(--color-text);\n  word-break: break-word;\n  font-family: inherit;\n}\n\n.toast-close {\n  flex-shrink: 0;\n  width: 24px;\n  height: 24px;\n  display: flex;\n  align-items: center;\n  justify-content: center;\n  background: transparent;\n  border: none;\n  border-radius: 4px;\n  color: var(--color-grey-500);\n  font-size: 1.5rem;\n  line-height: 1;\n  cursor: pointer;\n  transition: all 0.2s ease;\n  padding: 0;\n  margin-left: 0.25rem;\n\n  &:hover {\n    background: var(--color-grey-800);\n    color: var(--color-text);\n  }\n\n  &:active {\n    transform: scale(0.95);\n  }\n}\n\n.toast-progress {\n  position: absolute;\n  bottom: 0;\n  left: 0;\n  right: 0;\n  height: 3px;\n  background: currentColor;\n  opacity: 0.3;\n  transform-origin: left;\n  transition: transform linear;\n}\n\n// Dark mode adjustments\nhtml[data-darkmode=true] {\n  .toast {\n    background: var(--color-grey-300);\n    box-shadow: 0 4px 12px rgba(0, 0, 0, 0.4), 0 0 0 1px rgba(255, 255, 255, 0.05);\n  }\n\n  .toast-close:hover {\n    background: var(--color-grey-400);\n  }\n}\n\n// Mobile adjustments\n@media only screen and (max-width: 768px) {\n  .toast-container {\n    left: 50% !important;\n    right: auto !important;\n    top: 80px !important;\n    transform: translateX(-50%) !important;\n    align-items: center;\n\n    &.toast-bottom-right,\n    &.toast-bottom-center,\n    &.toast-bottom-left {\n      top: auto !important;\n      bottom: 80px !important;\n    }\n  }\n\n  .toast {\n    min-width: auto;\n    max-width: none;\n    width: 80vw;\n    transform: translateY(-100px);\n\n    &.toast-show {\n      transform: translateY(0);\n    }\n\n    &.toast-hide {\n      transform: translateY(-100px) scale(0.95);\n    }\n  }\n}\n\n// Accessibility\n@media (prefers-reduced-motion: reduce) {\n  .toast {\n    transition: opacity 0.2s ease;\n    transform: none !important;\n\n    &.toast-show {\n      opacity: 1;\n    }\n\n    &.toast-hide {\n      opacity: 0;\n    }\n  }\n}\n"
  },
  {
    "path": "changedetectionio/static/styles/scss/parts/_variables.scss",
    "content": "/**\n * CSS custom properties (aka variables).\n */\n\n:root {\n  --color-white: #fff;\n  --color-grey-50: #111;\n  --color-grey-100: #262626;\n  --color-grey-200: #333;\n  --color-grey-300: #444;\n  --color-grey-325: #555;\n  --color-grey-350: #565d64;\n  --color-grey-400: #666;\n  --color-grey-500: #777;\n  --color-grey-600: #999;\n  --color-grey-700: #cbcbcb;\n  --color-grey-750: #ddd;\n  --color-grey-800: #e0e0e0;\n  --color-grey-850: #eee;\n  --color-grey-900: #f2f2f2;\n  --color-black: #000;\n  --color-dark-red: #a00;\n  --color-light-red: #dd0000;\n\n  --color-background-page: var(--color-grey-100);\n  --color-background-gradient-first: #5ad8f7;\n  --color-background-gradient-second: #2f50af;\n  --color-background-gradient-third: #9150bf;\n  --color-background: var(--color-white);\n  --color-text: var(--color-grey-200);\n  --color-link: #1b98f8;\n  --color-menu-accent: #ed5900;\n  --color-background-code: var(--color-grey-850);\n  --color-error: var(--color-dark-red);\n  --color-error-input: #ffebeb;\n  --color-error-list: var(--color-light-red);\n  --color-table-background: var(--color-background);\n  --color-table-stripe: var(--color-grey-900);\n  --color-text-tab: var(--color-white);\n  --color-background-tab: rgba(255, 255, 255, 0.2);\n  --color-background-tab-hover: rgba(255, 255, 255, 0.5);\n  --color-text-tab-active: #222;\n  --color-api-key: #0078e7;\n\n  --color-background-button-primary: #0078e7;\n  --color-background-button-green: #42dd53;\n  --color-background-button-red: #dd4242;\n  --color-background-button-success: rgb(28, 184, 65);\n  --color-background-button-error: rgb(202, 60, 60);\n  --color-text-button-error: var(--color-white);\n  --color-background-button-warning: rgb(202, 60, 60);\n  --color-text-button-warning: var(--color-white);\n  --color-background-button-secondary: rgb(66, 184, 221);\n  --color-background-button-cancel: rgb(200, 200, 200);\n  --color-text-button: var(--color-white);\n  --color-background-button-tag: rgb(99, 99, 99);\n  --color-background-snapshot-age: #dfdfdf;\n  --color-error-text-snapshot-age: var(--color-white);\n  --color-error-background-snapshot-age: #ff0000;\n  --color-background-button-tag-active: #9c9c9c;\n\n  --color-text-messages: var(--color-white);\n  --color-background-messages-message: rgba(255, 255, 255, .2);\n  --color-background-messages-error: rgba(255, 1, 1, .5);\n  --color-background-messages-notice: rgba(255, 255, 255, .5);\n  --color-border-notification: #ccc;\n\n  --color-background-checkbox-operations: rgba(0, 0, 0, 0.05);\n  --color-warning: #ff3300;\n  --color-border-warning: var(--color-warning);\n  --color-text-legend: var(--color-white);\n\n  --color-link-new-version: #e07171;\n  --color-last-checked: #bbb;\n  --color-text-footer: #444;\n  --color-border-watch-table-cell: #eee;\n\n  --color-text-watch-tag-list: rgba(231, 0, 105, 0.4);\n  --color-background-new-watch-form: rgba(0, 0, 0, 0.05);\n  --color-background-new-watch-input: var(--color-white);\n  --color-background-new-watch-input-transparent: rgba(255, 255, 255, 0.1);\n  --color-text-new-watch-input: var(--color-text);\n\n  --color-border-input: var(--color-grey-500);\n  --color-shadow-input: var(--color-grey-400);\n  --color-background-input: var(--color-white);\n  --color-text-input: var(--color-text);\n  --color-text-input-description: var(--color-grey-500);\n  --color-text-input-placeholder: var(--color-grey-600);\n\n  --color-background-table-thead: var(--color-grey-800);\n  --color-border-table-cell: var(--color-grey-700);\n\n  --color-text-menu-heading: var(--color-grey-350);\n  --color-text-menu-link: var(--color-grey-500);\n  --color-background-menu-link-hover: var(--color-grey-850);\n  --color-text-menu-link-hover: var(--color-grey-300);\n\n  --color-shadow-jump: var(--color-grey-500);\n  --color-icon-github: var(--color-black);\n  --color-icon-github-hover: var(--color-grey-300);\n\n  --color-watch-table-error: var(--color-dark-red);\n  --color-watch-table-row-text: var(--color-grey-100);\n\n  --highlight-trigger-text-bg-color: #1b98f8;\n  --highlight-ignored-text-bg-color: var(--color-grey-700);\n  --highlight-blocked-text-bg-color: rgb(202, 60, 60);\n}\n\nhtml[data-darkmode=\"true\"] {\n  --color-link: #59bdfb;\n  --color-text: var(--color-white);\n\n  --color-background-gradient-first: #3f90a5;\n  --color-background-gradient-second: #1e316c;\n  --color-background-gradient-third: #4d2c64;\n\n  --color-background-new-watch-input: var(--color-grey-100);\n  --color-background-new-watch-input-transparent: var(--color-grey-100);\n  --color-text-new-watch-input: var(--color-text);\n  --color-background-table-thead: var(--color-grey-200);\n  --color-table-background: var(--color-grey-300);\n  --color-table-stripe: var(--color-grey-325);\n  --color-background: var(--color-grey-300);\n  --color-text-menu-heading: var(--color-grey-850);\n  --color-text-menu-link: var(--color-grey-800);\n  --color-border-table-cell: var(--color-grey-400);\n  --color-text-tab-active: var(--color-text);\n\n  --color-border-input: var(--color-grey-400);\n  --color-shadow-input: var(--color-grey-50);\n  --color-background-input: var(--color-grey-350);\n  --color-text-input-description: var(--color-grey-600);\n  --color-text-input-placeholder: var(--color-grey-600);\n  --color-text-watch-tag-list: rgba(250, 62, 146, 0.4);\n  --color-background-code: var(--color-grey-200);\n\n  --color-background-tab: rgba(0, 0, 0, 0.2);\n  --color-background-tab-hover: rgba(0, 0, 0, 0.5);\n\n  --color-background-snapshot-age: var(--color-grey-200);\n  --color-shadow-jump: var(--color-grey-200);\n  --color-icon-github: var(--color-white);\n  --color-icon-github-hover: var(--color-grey-700);\n  --color-watch-table-error: var(--color-light-red);\n  --color-watch-table-row-text: var(--color-grey-800);\n\n\n  .icon-spread {\n    filter: hue-rotate(-10deg) brightness(1.5);\n  }\n\n  .watch-table {\n\n    .title-col a[target=\"_blank\"]::after,\n    .current-diff-url::after {\n      filter: invert(.5) hue-rotate(10deg) brightness(2);\n    }\n\n    .status-browsersteps {\n      filter: invert(.5) hue-rotate(10deg) brightness(1.5);\n    }\n\n    .watch-controls {\n      .state-off {\n        img {\n          opacity: 0.3;\n        }\n      }\n      .state-on {\n        img {\n          opacity: 1.0;\n        }\n      }\n    }\n\n    .unviewed {\n      color: #fff;\n      &.error {\n        color: var(--color-watch-table-error);\n      }\n    }\n  }\n}\n"
  },
  {
    "path": "changedetectionio/static/styles/scss/parts/_visualselector.scss",
    "content": "\n#selector-wrapper {\n  height: 100%;\n  text-align: center;\n  \n  max-height: 70vh;\n  overflow-y: scroll;\n  position: relative;\n\n  //width: 100%;\n  >img {\n    position: absolute;\n    z-index: 4;\n    max-width: 100%;\n  }\n\n  >canvas {\n    position: relative;\n    z-index: 5;\n    max-width: 100%;\n\n    &:hover {\n      cursor: pointer;\n    }\n  }\n}\n\n#selector-current-xpath {\n  font-size: 80%;\n}"
  },
  {
    "path": "changedetectionio/static/styles/scss/parts/_watch_table-mobile.scss",
    "content": "$grid-col-checkbox: 20px;\n$grid-col-watch: 100px;\n$grid-gap: 0.5rem;\n\n\n@media (max-width: 767px) {\n\n  /*\n  Max width before this PARTICULAR table gets nasty\n  This query will take effect for any screen smaller than 760px\n  and also iPads specifically.\n  */\n  .watch-table {\n    /* make headings work on mobile */\n    thead {\n      display: block;\n\n      tr {\n        th {\n          display: inline-block;\n          // Hide the \"Last\" text for smaller screens\n          @media (max-width: 768px) {\n            .hide-on-mobile {\n              display: none;\n            }\n          }\n        }\n      }\n\n      .empty-cell {\n        display: none;\n      }\n    }\n\n\n    .last-checked {\n      margin-left: calc($grid-col-checkbox + $grid-gap);\n\n      > span {\n        vertical-align: middle;\n      }\n    }\n\n    .last-changed {\n      margin-left: calc($grid-col-checkbox + $grid-gap);\n    }\n\n    .last-checked::before {\n      color: var(--color-text);\n      content: \"Last Checked \";\n    }\n\n    .last-changed::before {\n      color: var(--color-text);\n      content: \"Last Changed \";\n    }\n\n    /* Force table to not be like tables anymore */\n    td.inline {\n      display: inline-block;\n    }\n\n    .pure-table td,\n    .pure-table th {\n      border: none;\n    }\n\n    td {\n      /* Behave  like a \"row\" */\n      border: none;\n      border-bottom: 1px solid var(--color-border-watch-table-cell);\n      vertical-align: middle;\n\n      &:before {\n        /* Top/left values mimic padding */\n        top: 6px;\n        left: 6px;\n        width: 45%;\n        padding-right: 10px;\n        white-space: nowrap;\n      }\n    }\n\n    &.pure-table-striped {\n      tr {\n        background-color: var(--color-table-background);\n      }\n\n      tr:nth-child(2n-1) {\n        background-color: var(--color-table-stripe);\n      }\n\n      tr:nth-child(2n-1) td {\n        background-color: inherit;\n      }\n    }\n  }\n}\n\n@media (max-width: 767px) {\n  .watch-table {\n    tbody {\n      tr {\n        padding-bottom: 10px;\n        padding-top: 10px;\n        display: grid;\n        grid-template-columns: $grid-col-checkbox 1fr $grid-col-watch;\n        grid-template-rows: auto auto auto auto;\n        gap: $grid-gap;\n\n        .counter-i {\n          display: none;\n        }\n\n        td.checkbox-uuid {\n          display: grid;\n          place-items: center;\n        }\n\n        td.inline {\n          /* display: block !important;;*/\n        }\n\n        > td {\n          border-bottom: none;\n        }\n\n        // Empty state message - span full width on mobile\n        > td[colspan] {\n          grid-column: 1 / -1;\n        }\n\n        > td.title-col {\n          grid-column: 1 / -1;\n          grid-row: 1;\n          .watch-title {\n            font-size: 0.92rem;\n          }\n          .link-spread {\n            display: none;\n          }\n        }\n\n        > td.last-checked {\n          grid-column: 1 / -1;\n          grid-row: 2;\n        }\n\n        > td.last-changed {\n          grid-column: 1 / -1;\n          grid-row: 3;\n        }\n\n        > td.checkbox-uuid {\n          grid-column: 1;\n          grid-row: 4;\n        }\n\n        > td.buttons {\n          grid-column: 2;\n          grid-row: 4;\n          display: flex;\n          align-items: center;\n          justify-content: flex-start;\n        }\n\n        > td.watch-controls {\n          grid-column: 3;\n          grid-row: 4;\n          display: grid;\n          place-items: center;\n\n          a img {\n            padding: 10px;\n          }\n        }\n      }\n    }\n  }\n  .pure-table td {\n    padding: 3px !important;\n  }\n}"
  },
  {
    "path": "changedetectionio/static/styles/scss/parts/_watch_table.scss",
    "content": "/* table related */\n#stats_row {\n  display: flex;\n  align-items: center;\n  width: 100%;\n  color: #fff;\n  font-size: 0.85rem;\n  >* {\n    padding-bottom: 0.5rem;\n  }\n  .left {\n    text-align: left;\n  }\n\n  .right {\n    opacity: 0.5;\n    transition: opacity 0.6s ease;\n    margin-left: auto; /* pushes it to the far right */\n    text-align: right;\n  }\n}\nbody.has-queue {\n  #stats_row {\n    .right {\n      opacity: 1.0;\n    }\n  }\n}\n\n.watch-table {\n  width: 100%;\n  font-size: 80%;\n\n  tr {\n    &.unviewed {\n      font-weight: bold;\n    }\n\n    color: var(--color-watch-table-row-text);\n  }\n\n\n  td {\n    white-space: nowrap;\n\n    &.title-col {\n      word-break: break-all;\n      white-space: normal;\n    }\n\n    a.external::after {\n      content: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAQElEQVR42qXKwQkAIAxDUUdxtO6/RBQkQZvSi8I/pL4BoGw/XPkh4XigPmsUgh0626AjRsgxHTkUThsG2T/sIlzdTsp52kSS1wAAAABJRU5ErkJggg==);\n      margin: 0 3px 0 5px;\n    }\n\n  }\n\n\n  th {\n    white-space: nowrap;\n\n    a {\n      font-weight: normal;\n\n      &.active {\n        font-weight: bolder;\n      }\n\n      &.inactive {\n        .arrow {\n          display: none;\n        }\n      }\n    }\n  }\n\n  /* Row with 'checking-now' */\n  tr.checking-now {\n    td:first-child {\n      position: relative;\n    }\n\n    td:first-child::before {\n      content: \"\";\n      position: absolute;\n      top: 0;\n      bottom: 0;\n      left: 0;\n      width: 3px;\n      background-color: #293eff;\n    }\n\n    td.last-checked {\n      .spinner-wrapper {\n        display: inline-block !important;\n      }\n\n      .innertext {\n        display: none !important;\n      }\n    }\n  }\n\n  tr.queued {\n    a.recheck {\n      display: none !important;\n    }\n\n    a.already-in-queue-button {\n      display: inline-block !important;\n    }\n  }\n\n  tr.paused {\n    a.pause-toggle {\n      &.state-on {\n        display: inline !important;\n      }\n\n      &.state-off {\n        display: none !important;\n      }\n    }\n  }\n\n  tr.notification_muted {\n    a.mute-toggle {\n      &.state-on {\n        display: inline !important;\n      }\n\n      &.state-off {\n        display: none !important;\n      }\n    }\n  }\n\n\n  tr.has-error {\n    color: var(--color-watch-table-error);\n\n    .error-text {\n      display: block !important;\n    }\n  }\n\n  tr.single-history {\n    a.preview-link {\n      display: inline-block !important;\n    }\n  }\n\n  tr.multiple-history {\n    a.history-link {\n      display: inline-block !important;\n    }\n  }\n\n\n}\n\n#watch-table-wrapper {\n  /* general styling */\n  #post-list-buttons {\n    text-align: right;\n    padding: 0px;\n    margin: 0px;\n\n    li {\n      display: inline-block;\n    }\n\n    a {\n      border-top-left-radius: initial;\n      border-top-right-radius: initial;\n      border-bottom-left-radius: 5px;\n      border-bottom-right-radius: 5px;\n    }\n  }\n\n  /* post list dynamically on/off stuff */\n\n  &.has-error {\n    #post-list-buttons {\n      #post-list-with-errors {\n        display: inline-block !important;\n      }\n    }\n  }\n\n  &.has-unread-changes {\n    #post-list-buttons {\n      #post-list-unread, #post-list-mark-views, #post-list-unread {\n        display: inline-block !important;\n      }\n    }\n  }\n}\n"
  },
  {
    "path": "changedetectionio/static/styles/scss/parts/_widgets.scss",
    "content": "\n// Ternary radio button group component\n.ternary-radio-group {\n  display: flex;\n  gap: 0;\n  border: 1px solid var(--color-grey-750);\n  border-radius: 4px;\n  overflow: hidden;\n  width: fit-content;\n  background: var(--color-background);\n\n  .ternary-radio-option {\n    position: relative;\n    cursor: pointer;\n    margin: 0;\n    display: flex;\n    align-items: center;\n\n    input[type=\"radio\"] {\n      position: absolute;\n      opacity: 0;\n      width: 0;\n      height: 0;\n    }\n\n    .ternary-radio-label {\n      padding: 8px 16px;\n      background: var(--color-grey-900);\n      border: none;\n      border-right: 1px solid var(--color-grey-750);\n      font-size: 13px;\n      font-weight: 500;\n      color: var(--color-text);\n      transition: all 0.2s ease;\n      cursor: pointer;\n      display: block;\n      text-align: center;\n    }\n\n    &:last-child .ternary-radio-label {\n      border-right: none;\n    }\n\n    input:checked + .ternary-radio-label {\n      background: var(--color-link);\n      color: var(--color-text-button);\n      font-weight: 600;\n\n      &.ternary-default {\n        background: var(--color-grey-600);\n        color: var(--color-text-button);\n      }\n\n      &:hover {\n        background: #1a7bc4;\n\n        &.ternary-default {\n          background: var(--color-grey-500);\n        }\n      }\n    }\n\n    &:hover .ternary-radio-label {\n      background: var(--color-grey-800);\n    }\n  }\n\n  @media (max-width: 480px) {\n    width: 100%;\n\n    .ternary-radio-label {\n      flex: 1;\n      min-width: auto;\n    }\n  }\n}\n\n// Standard radio button styling\ninput[type=\"radio\"].pure-radio:checked + label,\ninput[type=\"radio\"].pure-radio:checked {\n  background: var(--color-link);\n  color: var(--color-text-button);\n}\n\nhtml[data-darkmode=\"true\"] {\n  .ternary-radio-group {\n    .ternary-radio-option {\n      .ternary-radio-label {\n        background: var(--color-grey-350);\n      }\n\n      &:hover .ternary-radio-label {\n        background: var(--color-grey-400);\n      }\n\n      input:checked + .ternary-radio-label {\n        background: var(--color-link);\n        color: var(--color-text-button);\n\n        &.ternary-default {\n          background: var(--color-grey-600);\n        }\n\n        &:hover {\n          background: #1a7bc4;\n\n          &.ternary-default {\n            background: var(--color-grey-500);\n          }\n        }\n      }\n    }\n  }\n}"
  },
  {
    "path": "changedetectionio/static/styles/scss/styles.scss",
    "content": "/*\n * -- BASE STYLES --\n */\n\n@use \"settings\" as *;\n@use \"parts/variables\";\n@use \"parts/arrows\";\n@use \"parts/browser-steps\";\n@use \"parts/extra_proxies\";\n@use \"parts/extra_browsers\";\n@use \"parts/pagination\";\n@use \"parts/spinners\";\n@use \"parts/darkmode\";\n@use \"parts/menu\";\n@use \"parts/love\";\n@use \"parts/preview_text_filter\";\n@use \"parts/watch_table\";\n@use \"parts/watch_table-mobile\";\n@use \"parts/edit\";\n@use \"parts/conditions_table\";\n@use \"parts/lister_extra\";\n@use \"parts/socket\";\n@use \"parts/visualselector\";\n@use \"parts/widgets\";\n@use \"parts/diff_image\";\n@use \"parts/modal\";\n@use \"parts/language\";\n@use \"parts/action_sidebar\";\n@use \"parts/hamburger_menu\";\n@use \"parts/search_modal\";\n@use \"parts/notification_bubble\";\n@use \"parts/toast\";\n@use \"parts/login_form\";\n@use \"parts/tabs\";\n\n// Smooth transitions for theme switching\nbody,\n.pure-table,\n.pure-table thead,\n.pure-table td,\n.pure-table th,\n.pure-form input,\n.pure-form textarea,\n.pure-form select,\n.edit-form .inner,\n.pure-menu-horizontal,\nfooter,\n.sticky-tab,\n#diff-jump,\n.button-tag,\n#new-watch-form,\n#new-watch-form input:not(.pure-button),\ncode,\n.messages li,\n#checkbox-operations,\n.inline-warning,\na,\n.watch-controls img {\n  transition: color 0.4s ease, background-color 0.4s ease, background 0.4s ease, border-color 0.4s ease, box-shadow 0.4s ease;\n}\n\nbody {\n  color: var(--color-text);\n  background: var(--color-background-page);\n  font-family: Helvetica Neue, Helvetica, Lucida Grande, Arial, Ubuntu, Cantarell, Fira Sans, sans-serif;\n}\n\n.visually-hidden {\n  clip: rect(0 0 0 0);\n  clip-path: inset(50%);\n  height: 1px;\n  overflow: hidden;\n  position: absolute;\n  white-space: nowrap;\n  width: 1px;\n}\n\n// Row icons like chrome, pdf, share, etc\n.status-icon {\n  display: inline-block;\n  height: 1rem;\n  vertical-align: middle;\n}\n\n.pure-table-even {\n  background: var(--color-background);\n}\n\n/* Some styles from https://css-tricks.com/ */\na {\n  text-decoration: none;\n  color: var(--color-link);\n}\n\na.github-link {\n  color: var(--color-icon-github);\n  margin: 0 1rem 0 0.5rem;\n\n  svg {\n    fill: currentColor;\n  }\n\n  &:hover {\n    color: var(--color-icon-github-hover);\n  }\n}\n\n#search-result-info {\n  color: #fff;\n}\n\nbutton.toggle-button {\n  vertical-align: middle;\n  background: transparent;\n  border: none;\n  cursor: pointer;\n\n  color: var(--color-icon-github);\n\n  &:hover {\n    color: var(--color-icon-github-hover);\n  }\n\n  svg {\n    fill: currentColor;\n  }\n\n  .icon-light {\n    display: block;\n  }\n\n\n}\n\n.pure-menu-horizontal {\n  background: var(--color-background);\n  padding: 5px;\n  display: flex;\n  justify-content: space-between;\n  align-items: center;\n}\n\n#pure-menu-horizontal-spinner {\n  height: 3px;\n  background: linear-gradient(-75deg, #ff6000, #ff8f00, #ffdd00, #ed0000);\n  background-size: 400% 400%;\n  width: 100%;\n  animation: gradient 200s ease infinite;\n}\n\nbody.spinner-active {\n  #pure-menu-horizontal-spinner {\n    animation: gradient 1s ease infinite;\n  }\n}\n\n@keyframes gradient {\n\t0% {\n\t\tbackground-position: 0% 50%;\n\t}\n\t50% {\n\t\tbackground-position: 100% 50%;\n\t}\n\t100% {\n\t\tbackground-position: 0% 50%;\n\t}\n}\n.pure-menu-heading {\n  color: var(--color-text-menu-heading);\n}\n\n.pure-menu-link {\n  color: var(--color-text-menu-link);\n\n  &:hover {\n    background-color: var(--color-background-menu-link-hover);\n    color: var(--color-text-menu-link-hover);\n  }\n}\n\n\n.tab-pane-inner {\n  // .tab-pane-inner will have the #id that the tab button jumps/anchors to\n  scroll-margin-top: 200px;\n}\n\nsection.content {\n  @media only screen and (max-width: $desktop-wide-breakpoint) {\n    padding-top: 80px;\n  }\n  @media only screen and (min-width: $desktop-wide-breakpoint) {\n    padding-top: 100px;\n  }\n\n  padding-bottom: 1em;\n  flex-direction: column;\n  display: flex;\n  align-items: center;\n  justify-content: center;\n}\n\ncode {\n  background: var(--color-background-code);\n  color: var(--color-text);\n}\n\n.inline-tag {\n  white-space: nowrap;\n  border-radius: 5px;\n  padding: 2px 5px;\n  margin-right: 4px;\n  line-height: 1.2rem;\n}\n\n/* Processor type badges - colors auto-generated from processor names */\n.processor-badge {\n  @extend .inline-tag;\n  font-weight: 900;\n}\n\n.watch-tag-list {\n  color: var(--color-white);\n  background: var(--color-text-watch-tag-list);\n  @extend .inline-tag;\n  \n  /* Remove default anchor styling when used as links */\n  text-decoration: none;\n  \n  &:hover {\n    text-decoration: none;\n    opacity: 0.8;\n    cursor: pointer;\n  }\n  \n  &:visited {\n    color: var(--color-white);\n  }\n}\n\n@media (min-width: 768px) {\n  .box {\n    margin: 0 1em !important;\n  }\n}\n\n.box {\n  max-width: 100%;\n  margin: 0 0.3em;\n  flex-direction: column;\n  display: flex;\n  justify-content: center;\n}\n\n\nbody:after {\n  content: \"\";\n  background: linear-gradient(130deg, var(--color-background-gradient-first), var(--color-background-gradient-second) 41.07%, var(--color-background-gradient-third) 84.05%);\n}\n\nbody:after,\nbody:before {\n  display: block;\n  height: 650px;\n  position: absolute;\n  top: 0;\n  left: 0;\n  width: 100%;\n  z-index: -1;\n}\n\nbody::after {\n  opacity: 0.91;\n}\n\nbody::before {\n  // background-image set in base.html so it works with reverse proxies etc\n  content: \"\";\n}\n\nbody:after,\nbody:before {\n  -webkit-clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%);\n  clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%)\n}\n\n.button-small {\n  font-size: 85%;\n}\n\n.button-xsmall {\n  font-size: 70%;\n}\n\n.fetch-error {\n  padding-top: 1em;\n  font-size: 80%;\n  max-width: 400px;\n  display: block;\n}\n\n.pure-button-primary,\na.pure-button-primary,\n.pure-button-selected,\na.pure-button-selected {\n  background-color: var(--color-background-button-primary);\n}\n\n.button-secondary {\n  color: var(--color-text-button);\n  border-radius: 4px;\n  text-shadow: 0 1px 1px rgba(0, 0, 0, 0.2);\n}\n\n.button-success {\n  background: var(--color-background-button-success);\n}\n\n.button-tag {\n  background: var(--color-background-button-tag);\n  color: var(--color-text-button);\n  font-size: 65%;\n  border-bottom-left-radius: initial;\n  border-bottom-right-radius: initial;\n  margin-right: 4px;\n  &.active {\n    background: var(--color-background-button-tag-active);\n    font-weight: bold;\n  }\n\n}\n\n.button-error {\n  background: var(--color-background-button-error);\n  color: var(--color-text-button-error);\n}\n\n.button-warning {\n  background: var(--color-background-button-warning);\n  color: var(--color-text-button-warning);\n}\n\n.button-secondary {\n  background: var(--color-background-button-secondary);\n}\n\n.button-cancel {\n  background: var(--color-background-button-cancel);\n}\n\n.messages {\n  li {\n    list-style: none;\n    padding: 1em;\n    border-radius: 10px;\n    color: var(--color-text-messages);\n    font-weight: bold;\n\n    &.message {\n      background: var(--color-background-messages-message);\n    }\n\n    &.error {\n      background: var(--color-background-messages-error);\n    }\n\n    &.notice {\n      background: var(--color-background-messages-notice);\n    }\n  }\n\n  &.with-share-link {\n    >*:hover {\n      cursor: pointer;\n    }\n  }\n}\n\n.notifications-wrapper {\n  padding-top: 0.5rem;\n  #notification-test-log {\n    margin-top: 1rem;\n    padding: 1rem;\n    white-space: pre-wrap;\n    word-break: break-word;\n    overflow-wrap: break-word;\n    max-width: 100%;\n    box-sizing: border-box;\n    max-height: 12rem;\n    overflow-y: scroll;\n    border: 1px solid var(--color-border-notification);\n    border-radius: 5px;\n\n  }\n}\n\nlabel {\n &:hover {\n   cursor: pointer;\n }  \n}\n\n.grey-form-border {\n  border: 1px solid var(--color-border-notification);\n  padding: 0.5rem;\n  border-radius: 5px;\n}\n\n#notification-error-log {\n  border: 1px solid var(--color-border-notification);\n  padding: 1rem;\n  border-radius: 5px;\n  overflow-wrap: break-word;\n}\n\n#token-table {\n\n  &.pure-table td,\n  &.pure-table th {\n    font-size: 80%;\n  }\n}\n\n// Some field colouring for transperant field\n.pure-form input[type=text].transparent-field {\n  background-color:  var(--color-background-new-watch-input-transparent) !important;\n  color: var(--color-white) !important;\n  border: 1px solid rgba(255, 255, 255, 0.2) !important;\n  box-shadow: none !important;\n  -webkit-box-shadow: none !important;\n  &::placeholder {\n    opacity: 0.5;\n    color: rgba(255, 255, 255, 0.7);\n    font-weight: lighter;\n  }\n}\n\n#new-watch-form {\n  background: var(--color-background-new-watch-form);\n  padding: 1em;\n  border-radius: 10px;\n  margin-bottom: 1em;\n  max-width: 100%;\n\n  #url {\n    &::placeholder {\n      font-weight: bold;\n    }\n  }\n\n  input {\n    display: inline-block;\n    margin-bottom: 5px;\n  }\n\n  input:not(.pure-button) {\n    background-color: var(--color-background-new-watch-input);\n    color: var(--color-text-new-watch-input);\n  }\n\n  .label {\n    display: none;\n  }\n\n  legend {\n    color: var(--color-text-legend);\n    font-weight: bold;\n  }\n\n\n  #watch-add-wrapper-zone {\n    @media only screen and (min-width: 760px) {\n      display: flex;\n      gap: 0.3rem;\n      flex-direction: row;\n      min-width: 70vw;\n    }\n    /* URL field grows always, other stay static in width */\n    > span {\n      flex-grow: 0;\n\n      input {\n        width: 100%;\n        padding-right: 1em;\n      }\n\n      &:first-child {\n        flex-grow: 1;\n      }\n    }\n\n    @media only screen and (max-width: 760px) {\n      #url {\n        width: 100%;\n      }\n    }\n  }\n\n  #watch-group-tag {\n    font-size: 0.9rem;\n    padding: 0.3rem;\n    display: flex;\n    align-items: center;\n    gap: 0.5rem;\n    color: var(--color-white);\n    label, input {\n      margin: 0;\n    }\n\n    input {\n      flex: 1;\n    }\n  }\n}\n\n\n#diff-col {\n  padding-left: 40px;\n}\n\n#diff-jump {\n  position: fixed;\n  left: 0px;\n  top: 120px;\n  background: var(--color-background);\n  padding: 10px;\n  border-top-right-radius: 5px;\n  border-bottom-right-radius: 5px;\n  box-shadow: 1px 1px 4px var(--color-shadow-jump);\n\n  a {\n    color: var(--color-link);\n    cursor: pointer;\n    -moz-user-select: none;\n    -webkit-user-select: none;\n    -ms-user-select: none;\n    user-select: none;\n    -o-user-select: none;\n  }\n}\n\nfooter {\n  padding: 10px;\n  background: var(--color-background);\n  color: var(--color-text-footer);\n  text-align: center;\n}\n\n#feed-icon {\n  vertical-align: middle;\n}\n\n#top-right-menu {\n  // Just let flex overflow the x axis for now\n  /*\n      position: absolute;\n      right: 0px;\n      background: linear-gradient(to right, #fff0, #fff 10%);\n      padding-left: 20px;\n      padding-right: 10px;\n      */\n}\n\n.sticky-tab {\n  @media only screen and (max-width: $desktop-wide-breakpoint) {\n    display: none;\n  }\n  position: absolute;\n  top: 60px;\n  font-size: 65%;\n  background: var(--color-background);\n  padding: 10px;\n\n  &#left-sticky {\n    left: 0;\n    position: fixed;\n    border-top-right-radius: 5px;\n    border-bottom-right-radius: 5px;\n    box-shadow: 1px 1px 4px var(--color-shadow-jump);\n  }\n\n  &#right-sticky {\n    right: 0px;\n  }\n\n  &#hosted-sticky {\n    right: 0px;\n    top: 100px;\n    font-weight: bold;\n  }\n}\n\n#new-version-text a {\n  color: var(--color-link-new-version);\n}\n\n.watch-controls {\n  color: #f8321b;\n\n  .state-on {\n    img {\n      opacity: 0.8;\n    }\n  }\n\n  /* default */\n  img {\n    opacity: 0.2;\n  }\n\n  img {\n    &:hover {\n      transition: opacity 0.3s;\n      opacity: 0.8;\n    }\n  }\n}\n\n.monospaced-textarea {\n  textarea {\n    width: 100%;\n    font-family: monospace;\n    white-space: pre;\n    overflow-wrap: normal;\n    // No scrollbars until needed.\n    overflow-x: auto;\n  }\n}\n\n\n.pure-form {\n  fieldset {\n    padding-top: 0px;\n\n    ul {\n      padding-bottom: 0px;\n      margin-bottom: 0px;\n    }\n  }\n\n  .pure-control-group,\n  .pure-group,\n  .pure-controls {\n    padding-bottom: 1em;\n\n    div {\n      margin: 0px;\n    }\n\n    .checkbox {\n      >* {\n        display: inline;\n        vertical-align: middle;\n      }\n\n      >label {\n        padding-left: 5px;\n      }\n    }\n\n    legend {\n      color: var(--color-text-legend);\n    }\n  }\n\n  /* The input fields with errors */\n  .error {\n    input {\n      background-color: var(--color-error-input);\n    }\n  }\n\n  /* The list of errors */\n  ul.errors {\n    padding: .5em .6em;\n    border: 1px solid var(--color-error-list);\n    border-radius: 4px;\n    vertical-align: middle;\n    -webkit-box-sizing: border-box;\n    box-sizing: border-box;\n\n    li {\n      margin-left: 1em;\n      color: var(--color-error-list);\n    }\n  }\n\n  label {\n    font-weight: bold;\n  }\n\n  textarea {\n    width: 100%;\n  }\n\n  .inline-radio {\n    ul {\n      margin: 0px;\n      list-style: none;\n\n      li {\n        display: flex;\n        align-items: center;\n        gap: 1em;\n      }\n    }\n  }\n}\n\n\n@media only screen and (max-width: 760px),\n(min-device-width: 768px) and (max-device-width: $desktop-wide-breakpoint) {\n  .edit-form {\n    padding: 0.5em;\n    margin: 0;\n  }\n\n  #nav-menu {\n    overflow-x: scroll;\n  }\n}\n\n\n@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: $desktop-wide-breakpoint) {\n  input[type='text'] {\n    width: 100%;\n  }\n}\n\n.pure-table {\n  border-color: var(--color-border-table-cell);\n\n  thead {\n    background-color: var(--color-background-table-thead);\n    color: var(--color-text);\n    border-bottom: 1px solid var(--color-background-table-thead);\n  }\n\n  td,\n  th {\n    border-left-color: var(--color-border-table-cell);\n  }\n}\n\n.pure-table-striped {\n  tr:nth-child(2n-1) {\n    td {\n      background-color: var(--color-table-stripe);\n    }\n  }\n}\n\n.pure-form input[type=color],\n.pure-form input[type=date],\n.pure-form input[type=datetime-local],\n.pure-form input[type=datetime],\n.pure-form input[type=email],\n.pure-form input[type=month],\n.pure-form input[type=number],\n.pure-form input[type=password],\n.pure-form input[type=search],\n.pure-form input[type=tel],\n.pure-form input[type=text],\n.pure-form input[type=time],\n.pure-form input[type=url],\n.pure-form input[type=week],\n.pure-form select,\n.pure-form textarea {\n  border: var(--color-border-input);\n  box-shadow: inset 0 1px 3px var(--color-shadow-input);\n  background-color: var(--color-background-input);\n  color: var(--color-text-input);\n\n  &:active {\n    background-color: var(--color-background-input);\n  }\n}\n\ninput::placeholder,\ntextarea::placeholder {\n  color: var(--color-text-input-placeholder);\n}\n\n\n/** Desktop vs mobile input field strategy\n- We dont use 'size' with  because `size` is too unreliable to override, and will often push-out\n- Rely always on width in CSS\n*/\n/** Set max width for input field */\n.m-d {\n  min-width: 100%;\n}\n\n@media only screen and (min-width: 761px) {\n\n  /* m-d is medium-desktop */\n  .m-d {\n    min-width: 80%;\n  }\n}\n\n\n$form-edge-padding: 20px;\n\n.pure-form-stacked {\n  >div:first-child {\n    display: block;\n  }\n}\n\n// Login form styles moved to parts/_login_form.scss\n\n.tab-pane-inner {\n\n  &:not(:target) {\n    display: none;\n  }\n\n  &:target {\n    display: block;\n  }\n\n  // doesnt need padding because theres another row of buttons/activity\n  padding: 0px;\n}\n\n.beta-logo {\n  height: 50px;\n  // looks better when it's hanging off a little\n  right: -3px;\n  top: -3px;\n  position: absolute;\n}\n\n#selector-header {\n  padding-bottom: 1em;\n}\n\nbody.full-width {\n  .edit-form {\n    width: 95%;\n  }\n}\n\n.edit-form {\n  min-width: 70%;\n  /* so it cant overflow */\n  max-width: 95%;\n\n  .box-wrap {\n    position: relative;\n  }\n\n  .inner {\n    background: var(--color-background);\n    padding: $form-edge-padding;\n  }\n\n  #actions {\n    display: block;\n    background: var(--color-background);\n  }\n\n  /* Make action buttons have consistent size and spacing */\n  #actions .pure-control-group {\n    display: flex;\n    gap: 0.625em;\n    flex-wrap: wrap;\n  }\n\n  .pure-form-message-inline {\n    padding-left: 0;\n    color: var(--color-text-input-description);\n    code {\n      font-size: .875em;\n    }\n  }\n}\n\n.border-fieldset {\n  h3 {\n    margin-top: 0;\n  }\n  border: 1px solid #ccc;\n  padding: 1rem;\n  border-radius: 5px;\n  margin-bottom: 1rem;\n  fieldset:last-of-type {\n    padding-bottom: 0;\n    .pure-control-group {\n      padding-bottom: 0;\n    }\n  }\n}\n\n\n\nul {\n  padding-left: 1em;\n  padding-top: 0px;\n  margin-top: 4px;\n}\n\n.time-check-widget {\n  tr {\n    display: inline;\n\n    input[type=\"number\"] {\n      width: 5em;\n    }\n  }\n}\n\n@media only screen and (max-width: 760px) {\n  .time-check-widget {\n    tbody {\n      display: grid;\n      grid-template-columns: auto 1fr auto 1fr;\n      gap: 0.625em 0.3125em;\n      align-items: center;\n    }    \n    tr {\n      display: contents; \n      th {\n        text-align: right;\n        padding-right: 5px;\n      }\n      input[type=\"number\"] {\n        width: 100%;\n        max-width: 5em;\n      }\n    }\n  }\n}\n\n#webdriver_delay {\n    width: 5em;\n}\n\n#api-key {\n  &:hover {\n    cursor: pointer;\n  }\n}\n\n#api-key-copy {\n  color: var(--color-api-key);\n}\n\n.button-green {\n  background-color: var(--color-background-button-green);\n}\n\n.button-red {\n  background-color: var(--color-background-button-red);\n}\n\n.noselect {\n  -webkit-touch-callout: none;\n  /* iOS Safari */\n  -webkit-user-select: none;\n  /* Safari */\n  -moz-user-select: none;\n  /* Old versions of Firefox */\n  -ms-user-select: none;\n  /* Internet Explorer/Edge */\n  user-select: none;\n  /* Non-prefixed version, currently\n    supported by Chrome, Edge, Opera and Firefox */\n}\n\n\n#checkbox-operations {\n  background: var(--color-background-checkbox-operations);\n  padding: 1em;\n  border-radius: 10px;\n  margin-bottom: 1em;\n  display: none;\n  button {\n    /* some space if they wrap the page */\n    margin-bottom: 3px;\n    margin-top: 3px;\n    /* vertically center icon and text */\n    display: inline-flex;\n    align-items: center;\n  }\n}\n\n.checkbox-uuid {\n  >* {\n    vertical-align: middle;\n  }\n}\n\n.inline-warning {\n  >span {\n    display: inline-block;\n    vertical-align: middle;\n  }\n\n  img.inline-warning-icon {\n    display: inline;\n    height: 26px;\n    vertical-align: middle;\n  }\n\n  border: 1px solid var(--color-border-warning);\n  padding: 0.5rem;\n  border-radius: 5px;\n  color: var(--color-warning);\n}\n\n/* automatic price following helpers */\n.tracking-ldjson-price-data {\n  background-color: var(--color-background-button-green);\n  color: #000;\n  opacity: 0.6;\n  @extend .inline-tag;\n}\n\n.ldjson-price-track-offer {\n  a.pure-button {\n    border-radius: 3px;\n    padding: 3px;\n    background-color: var(--color-background-button-green);\n  }\n\n  font-weight: bold;\n  font-style: italic;\n}\n\n.price-follow-tag-icon {\n  display: inline-block;\n  height: 0.8rem;\n  vertical-align: middle;\n}\n\n\n#quick-watch-processor-type {\n  ul#processor {\n    color: #fff;\n    padding-left: 0px;\n    li {\n      list-style: none;\n      font-size: 0.9rem;\n      display: grid;\n      grid-template-columns: auto 1fr;\n      align-items: center;\n      gap: 0.5rem;\n      margin-bottom: 0.5rem;\n    }\n  }\n  label, input {\n    padding: 0;\n    margin: 0;\n  }\n}\n\n.restock-label {\n  &.in-stock {\n    background-color: var(--color-background-button-green);\n    color: #fff;\n  }\n  &.not-in-stock {\n    background-color: var(--color-background-button-cancel);\n    color: #777;\n  }\n  &.error {\n    background-color: var(--color-background-button-error);\n    color: #fff;\n    opacity: 0.7;\n  }\n\n  svg {\n    vertical-align: middle;\n  }\n\n  @extend .inline-tag;\n}\n\n#chrome-extension-link {\n  img {\n    height: 21px;\n    padding: 2px;\n    vertical-align: middle;\n  }\n\n  padding: 9px;\n  border: 1px solid var(--color-grey-800);\n  border-radius: 10px;\n  vertical-align: middle;\n}\n\n#realtime-conn-error {\n  position: fixed;\n  bottom: 0;\n  left: 0;\n  background: var(--color-warning);\n  padding: 10px;\n  font-size: 0.8rem;\n  color: #fff;\n  opacity: 0.8;\n}\n\n#bottom-horizontal-offscreen {\n  position: fixed;\n  bottom: 0;\n  left: 0;\n  right: 0;\n  width: 100%;\n  min-height: 50px;\n  max-height: 50vh; // Don't take more than 50% of viewport height\n  background: #ffffffb8;\n  border-top: 1px solid var(--color-border-table-cell);\n  padding: 10px;\n  box-shadow: 0 -2px 10px rgba(0, 0, 0, 0.2);\n  z-index: 100;\n  overflow-y: auto; // Allow scrolling if content exceeds max-height\n\n  // Smooth transition when shown/hidden\n  transition: opacity 0.3s ease-in-out;\n\n  // When JavaScript removes display:none, ensure it scrolls into view\n  scroll-margin-bottom: 10px;\n\n  // Center contents horizontally\n  display: flex;\n  justify-content: center;\n  align-items: center;\n}\n\nul#highlightSnippetActions {\n  list-style: none;\n  li {\n    display: inline-block;\n  }\n}\n\n\n"
  },
  {
    "path": "changedetectionio/static/styles/styles.css",
    "content": ":root{--color-white: #fff;--color-grey-50: #111;--color-grey-100: #262626;--color-grey-200: #333;--color-grey-300: #444;--color-grey-325: #555;--color-grey-350: #565d64;--color-grey-400: #666;--color-grey-500: #777;--color-grey-600: #999;--color-grey-700: #cbcbcb;--color-grey-750: #ddd;--color-grey-800: #e0e0e0;--color-grey-850: #eee;--color-grey-900: #f2f2f2;--color-black: #000;--color-dark-red: #a00;--color-light-red: #dd0000;--color-background-page: var(--color-grey-100);--color-background-gradient-first: #5ad8f7;--color-background-gradient-second: #2f50af;--color-background-gradient-third: #9150bf;--color-background: var(--color-white);--color-text: var(--color-grey-200);--color-link: #1b98f8;--color-menu-accent: #ed5900;--color-background-code: var(--color-grey-850);--color-error: var(--color-dark-red);--color-error-input: #ffebeb;--color-error-list: var(--color-light-red);--color-table-background: var(--color-background);--color-table-stripe: var(--color-grey-900);--color-text-tab: var(--color-white);--color-background-tab: rgba(255, 255, 255, 0.2);--color-background-tab-hover: rgba(255, 255, 255, 0.5);--color-text-tab-active: #222;--color-api-key: #0078e7;--color-background-button-primary: #0078e7;--color-background-button-green: #42dd53;--color-background-button-red: #dd4242;--color-background-button-success: rgb(28, 184, 65);--color-background-button-error: rgb(202, 60, 60);--color-text-button-error: var(--color-white);--color-background-button-warning: rgb(202, 60, 60);--color-text-button-warning: var(--color-white);--color-background-button-secondary: rgb(66, 184, 221);--color-background-button-cancel: rgb(200, 200, 200);--color-text-button: var(--color-white);--color-background-button-tag: rgb(99, 99, 99);--color-background-snapshot-age: #dfdfdf;--color-error-text-snapshot-age: var(--color-white);--color-error-background-snapshot-age: #ff0000;--color-background-button-tag-active: #9c9c9c;--color-text-messages: var(--color-white);--color-background-messages-message: rgba(255, 255, 255, .2);--color-background-messages-error: rgba(255, 1, 1, .5);--color-background-messages-notice: rgba(255, 255, 255, .5);--color-border-notification: #ccc;--color-background-checkbox-operations: rgba(0, 0, 0, 0.05);--color-warning: #ff3300;--color-border-warning: var(--color-warning);--color-text-legend: var(--color-white);--color-link-new-version: #e07171;--color-last-checked: #bbb;--color-text-footer: #444;--color-border-watch-table-cell: #eee;--color-text-watch-tag-list: rgba(231, 0, 105, 0.4);--color-background-new-watch-form: rgba(0, 0, 0, 0.05);--color-background-new-watch-input: var(--color-white);--color-background-new-watch-input-transparent: rgba(255, 255, 255, 0.1);--color-text-new-watch-input: var(--color-text);--color-border-input: var(--color-grey-500);--color-shadow-input: var(--color-grey-400);--color-background-input: var(--color-white);--color-text-input: var(--color-text);--color-text-input-description: var(--color-grey-500);--color-text-input-placeholder: var(--color-grey-600);--color-background-table-thead: var(--color-grey-800);--color-border-table-cell: var(--color-grey-700);--color-text-menu-heading: var(--color-grey-350);--color-text-menu-link: var(--color-grey-500);--color-background-menu-link-hover: var(--color-grey-850);--color-text-menu-link-hover: var(--color-grey-300);--color-shadow-jump: var(--color-grey-500);--color-icon-github: var(--color-black);--color-icon-github-hover: var(--color-grey-300);--color-watch-table-error: var(--color-dark-red);--color-watch-table-row-text: var(--color-grey-100);--highlight-trigger-text-bg-color: #1b98f8;--highlight-ignored-text-bg-color: var(--color-grey-700);--highlight-blocked-text-bg-color: rgb(202, 60, 60)}html[data-darkmode=true]{--color-link: #59bdfb;--color-text: var(--color-white);--color-background-gradient-first: #3f90a5;--color-background-gradient-second: #1e316c;--color-background-gradient-third: #4d2c64;--color-background-new-watch-input: var(--color-grey-100);--color-background-new-watch-input-transparent: var(--color-grey-100);--color-text-new-watch-input: var(--color-text);--color-background-table-thead: var(--color-grey-200);--color-table-background: var(--color-grey-300);--color-table-stripe: var(--color-grey-325);--color-background: var(--color-grey-300);--color-text-menu-heading: var(--color-grey-850);--color-text-menu-link: var(--color-grey-800);--color-border-table-cell: var(--color-grey-400);--color-text-tab-active: var(--color-text);--color-border-input: var(--color-grey-400);--color-shadow-input: var(--color-grey-50);--color-background-input: var(--color-grey-350);--color-text-input-description: var(--color-grey-600);--color-text-input-placeholder: var(--color-grey-600);--color-text-watch-tag-list: rgba(250, 62, 146, 0.4);--color-background-code: var(--color-grey-200);--color-background-tab: rgba(0, 0, 0, 0.2);--color-background-tab-hover: rgba(0, 0, 0, 0.5);--color-background-snapshot-age: var(--color-grey-200);--color-shadow-jump: var(--color-grey-200);--color-icon-github: var(--color-white);--color-icon-github-hover: var(--color-grey-700);--color-watch-table-error: var(--color-light-red);--color-watch-table-row-text: var(--color-grey-800)}html[data-darkmode=true] .icon-spread{filter:hue-rotate(-10deg) brightness(1.5)}html[data-darkmode=true] .watch-table .title-col a[target=_blank]::after,html[data-darkmode=true] .watch-table .current-diff-url::after{filter:invert(0.5) hue-rotate(10deg) brightness(2)}html[data-darkmode=true] .watch-table .status-browsersteps{filter:invert(0.5) hue-rotate(10deg) brightness(1.5)}html[data-darkmode=true] .watch-table .watch-controls .state-off img{opacity:.3}html[data-darkmode=true] .watch-table .watch-controls .state-on img{opacity:1}html[data-darkmode=true] .watch-table .unviewed{color:#fff}html[data-darkmode=true] .watch-table .unviewed.error{color:var(--color-watch-table-error)}.arrow{border:solid #1b98f8;border-width:0 2px 2px 0;display:inline-block;padding:3px}.arrow.right{transform:rotate(-45deg);-webkit-transform:rotate(-45deg)}.arrow.left{transform:rotate(135deg);-webkit-transform:rotate(135deg)}.arrow.up,.arrow.asc{transform:rotate(-135deg);-webkit-transform:rotate(-135deg)}.arrow.down,.arrow.desc{transform:rotate(45deg);-webkit-transform:rotate(45deg)}#browser_steps th{display:none}#browser_steps li{list-style:decimal;padding:5px}#browser_steps li.browser-step-with-error{background-color:#ffd6d6;border-radius:4px}#browser_steps li:not(:first-child):hover{opacity:1}#browser_steps li .control{padding-left:5px;padding-right:5px}#browser_steps li .control a{font-size:70%}#browser_steps li.empty{padding:0px;opacity:.35}#browser_steps li.empty .control{display:none}#browser_steps li:hover{background:#eee}#browser_steps li>label{display:none}@media only screen and (min-width: 760px){#browser-steps .flex-wrapper{display:flex;flex-flow:row;height:70vh;font-size:80%}#browser-steps .flex-wrapper #browser-steps-ui{flex-grow:1;flex-shrink:1;flex-basis:0;background-color:#eee;border-radius:5px}#browser-steps-fieldlist{flex-grow:0;flex-shrink:0;flex-basis:auto;max-width:400px;padding-left:1rem;overflow-y:scroll}#browsersteps-selector-wrapper{height:100% !important}}#browsersteps-selector-wrapper{width:100%;overflow-y:scroll;position:relative;height:80vh}#browsersteps-selector-wrapper>img{position:absolute;max-width:100%}#browsersteps-selector-wrapper>canvas{position:relative;max-width:100%}#browsersteps-selector-wrapper>canvas:hover{cursor:pointer}#browsersteps-selector-wrapper .loader{position:absolute;left:50%;top:50%;transform:translate(-50%, -50%);z-index:100;max-width:350px;text-align:center}#browsersteps-selector-wrapper .spinner,#browsersteps-selector-wrapper .spinner:after{width:80px;height:80px;font-size:3px}#browsersteps-selector-wrapper #browsersteps-click-start{color:var(--color-grey-400)}#browsersteps-selector-wrapper #browsersteps-click-start:hover{cursor:pointer}ul#requests-extra_proxies{list-style:none}ul#requests-extra_proxies li>label{display:none}ul#requests-extra_proxies table tr{display:table-row}ul#requests-extra_proxies table tr input[type=text]{width:100%}@media only screen and (min-width: 1024px){ul#requests-extra_proxies table tr{display:inline}}#request label[for=proxy]{display:inline-block}body.proxy-check-active #request .proxy-check-details{font-size:80%;color:#555;display:block;padding-left:2em;max-width:500px}body.proxy-check-active #request .proxy-timing{font-size:80%;padding-left:1rem;color:var(--color-link)}#recommended-proxy{display:grid;gap:2rem;padding-bottom:1em}@media(min-width: 991px){#recommended-proxy{grid-template-columns:repeat(2, 1fr)}}#recommended-proxy>div{border:1px #aaa solid;border-radius:4px;padding:1em}#extra-proxies-setting{border:1px solid var(--color-grey-800);border-radius:4px;margin:1em;padding:1em}ul#requests-extra_browsers{list-style:none}ul#requests-extra_browsers li>label{display:none}ul#requests-extra_browsers table tr{display:table-row}ul#requests-extra_browsers table tr input[type=text]{width:100%}@media only screen and (min-width: 1280px){ul#requests-extra_browsers table tr{display:inline}ul#requests-extra_browsers table tr input[type=text]{width:100%}}#extra-browsers-setting{border:1px solid var(--color-grey-800);border-radius:4px;margin:1em;padding:1em}.pagination-page-info{text-transform:capitalize}.pagination.menu>*{display:inline-block}.pagination.menu li{display:inline-block}.pagination.menu a{padding:.65rem;margin:3px;border:none;background:#444;border-radius:2px;color:var(--color-text-button)}.pagination.menu a.disabled{display:none}.pagination.menu a.active{font-weight:bold;background:#888}.pagination.menu a:hover{background:#999}.spinner,.spinner:after{border-radius:50%;width:10px;height:10px}.spinner{margin:0px auto;font-size:3px;vertical-align:middle;display:inline-block;text-indent:-9999em;border-top:1.1em solid rgba(38,104,237,.2);border-right:1.1em solid rgba(38,104,237,.2);border-bottom:1.1em solid rgba(38,104,237,.2);border-left:1.1em solid #2668ed;-webkit-transform:translateZ(0);-ms-transform:translateZ(0);transform:translateZ(0);-webkit-animation:load8 1.1s infinite linear;animation:load8 1.1s infinite linear}@-webkit-keyframes load8{0%{-webkit-transform:rotate(0deg);transform:rotate(0deg)}100%{-webkit-transform:rotate(360deg);transform:rotate(360deg)}}@keyframes load8{0%{-webkit-transform:rotate(0deg);transform:rotate(0deg)}100%{-webkit-transform:rotate(360deg);transform:rotate(360deg)}}.toggle-light-mode .icon-dark{display:none}html[data-darkmode=true] .toggle-light-mode .icon-light{display:none}html[data-darkmode=true] .toggle-light-mode .icon-dark{display:block}.pure-menu-link{padding:.5rem 1em;line-height:1.2rem}#menu-mute,#menu-pause{padding-left:.3rem;padding-right:.3rem}#menu-mute img,#menu-pause img{height:1.2rem}.pure-menu-item svg{height:1.2rem}.pure-menu-item *{vertical-align:middle}.pure-menu-item .github-link{height:1.8rem;display:block}.pure-menu-item .github-link svg{height:100%}.pure-menu-item .bi-heart:hover{cursor:pointer}.pure-menu-item.active .pure-menu-link{background-color:var(--color-background-menu-link-hover);color:var(--color-text-menu-link-hover)}#cdio-logo{padding-left:.5em}#inline-menu-extras-group>*{display:inline-block}#overlay{opacity:.95;position:fixed;width:350px;max-width:100%;height:100%;top:0;right:-350px;background-color:var(--color-table-stripe);z-index:2;transform:translateX(0);transition:transform .5s ease}#overlay.visible{transform:translateX(-100%)}#overlay .content{font-size:.875rem;padding:1rem;margin-top:5rem;max-width:400px;color:var(--color-watch-table-row-text)}#heartpath{transition:all ease .3s !important}#heartpath:hover{fill:red !important;transition:all ease .3s !important}.minitabs-wrapper{width:100%}.minitabs-wrapper>div[id]{padding:20px;border:1px solid #ccc;border-top:none}.minitabs-wrapper .minitabs-content{width:100%;display:flex}.minitabs-wrapper .minitabs-content>div{flex:1 1 auto;min-width:0;overflow:scroll}.minitabs-wrapper .minitabs{display:flex;border-bottom:1px solid #ccc}.minitabs-wrapper .minitab{flex:1;text-align:center;padding:12px 0;text-decoration:none;color:#333;background-color:#f1f1f1;border:1px solid #ccc;border-bottom:none;cursor:pointer;transition:background-color .3s}.minitabs-wrapper .minitab:hover{background-color:#ddd}.minitabs-wrapper .minitab.active{background-color:#fff;font-weight:bold}@media(min-width: 800px){body.preview-text-enabled #filters-and-triggers>div{display:flex;gap:20px;position:relative}}body.preview-text-enabled #edit-text-filter,body.preview-text-enabled #text-preview{flex:1;align-self:flex-start}body.preview-text-enabled #edit-text-filter #pro-tips{display:none}body.preview-text-enabled #text-preview{position:sticky;top:20px;padding-top:1rem;padding-bottom:1rem;display:block !important}body.preview-text-enabled #activate-text-preview{background-color:var(--color-grey-500)}body.preview-text-enabled .monospace-preview{background:var(--color-background-input);border:1px solid var(--color-grey-600);padding:1rem;color:var(--color-text-input);font-family:\"Courier New\",Courier,monospace;font-size:70%;word-break:break-word;white-space:pre-wrap}#activate-text-preview{right:0;position:absolute;z-index:3;box-shadow:1px 1px 4px var(--color-shadow-jump)}#stats_row{display:flex;align-items:center;width:100%;color:#fff;font-size:.85rem}#stats_row>*{padding-bottom:.5rem}#stats_row .left{text-align:left}#stats_row .right{opacity:.5;transition:opacity .6s ease;margin-left:auto;text-align:right}body.has-queue #stats_row .right{opacity:1}.watch-table{width:100%;font-size:80%}.watch-table tr{color:var(--color-watch-table-row-text)}.watch-table tr.unviewed{font-weight:bold}.watch-table td{white-space:nowrap}.watch-table td.title-col{word-break:break-all;white-space:normal}.watch-table td a.external::after{content:url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAQElEQVR42qXKwQkAIAxDUUdxtO6/RBQkQZvSi8I/pL4BoGw/XPkh4XigPmsUgh0626AjRsgxHTkUThsG2T/sIlzdTsp52kSS1wAAAABJRU5ErkJggg==);margin:0 3px 0 5px}.watch-table th{white-space:nowrap}.watch-table th a{font-weight:normal}.watch-table th a.active{font-weight:bolder}.watch-table th a.inactive .arrow{display:none}.watch-table tr.checking-now td:first-child{position:relative}.watch-table tr.checking-now td:first-child::before{content:\"\";position:absolute;top:0;bottom:0;left:0;width:3px;background-color:#293eff}.watch-table tr.checking-now td.last-checked .spinner-wrapper{display:inline-block !important}.watch-table tr.checking-now td.last-checked .innertext{display:none !important}.watch-table tr.queued a.recheck{display:none !important}.watch-table tr.queued a.already-in-queue-button{display:inline-block !important}.watch-table tr.paused a.pause-toggle.state-on{display:inline !important}.watch-table tr.paused a.pause-toggle.state-off{display:none !important}.watch-table tr.notification_muted a.mute-toggle.state-on{display:inline !important}.watch-table tr.notification_muted a.mute-toggle.state-off{display:none !important}.watch-table tr.has-error{color:var(--color-watch-table-error)}.watch-table tr.has-error .error-text{display:block !important}.watch-table tr.single-history a.preview-link{display:inline-block !important}.watch-table tr.multiple-history a.history-link{display:inline-block !important}#watch-table-wrapper #post-list-buttons{text-align:right;padding:0px;margin:0px}#watch-table-wrapper #post-list-buttons li{display:inline-block}#watch-table-wrapper #post-list-buttons a{border-top-left-radius:initial;border-top-right-radius:initial;border-bottom-left-radius:5px;border-bottom-right-radius:5px}#watch-table-wrapper.has-error #post-list-buttons #post-list-with-errors{display:inline-block !important}#watch-table-wrapper.has-unread-changes #post-list-buttons #post-list-unread,#watch-table-wrapper.has-unread-changes #post-list-buttons #post-list-mark-views,#watch-table-wrapper.has-unread-changes #post-list-buttons #post-list-unread{display:inline-block !important}@media(max-width: 767px){.watch-table thead{display:block}.watch-table thead tr th{display:inline-block}}@media(max-width: 767px)and (max-width: 768px){.watch-table thead tr th .hide-on-mobile{display:none}}@media(max-width: 767px){.watch-table thead .empty-cell{display:none}.watch-table .last-checked{margin-left:calc(20px + .5rem)}.watch-table .last-checked>span{vertical-align:middle}.watch-table .last-changed{margin-left:calc(20px + .5rem)}.watch-table .last-checked::before{color:var(--color-text);content:\"Last Checked \"}.watch-table .last-changed::before{color:var(--color-text);content:\"Last Changed \"}.watch-table td.inline{display:inline-block}.watch-table .pure-table td,.watch-table .pure-table th{border:none}.watch-table td{border:none;border-bottom:1px solid var(--color-border-watch-table-cell);vertical-align:middle}.watch-table td:before{top:6px;left:6px;width:45%;padding-right:10px;white-space:nowrap}.watch-table.pure-table-striped tr{background-color:var(--color-table-background)}.watch-table.pure-table-striped tr:nth-child(2n-1){background-color:var(--color-table-stripe)}.watch-table.pure-table-striped tr:nth-child(2n-1) td{background-color:inherit}}@media(max-width: 767px){.watch-table tbody tr{padding-bottom:10px;padding-top:10px;display:grid;grid-template-columns:20px 1fr 100px;grid-template-rows:auto auto auto auto;gap:.5rem}.watch-table tbody tr .counter-i{display:none}.watch-table tbody tr td.checkbox-uuid{display:grid;place-items:center}.watch-table tbody tr>td{border-bottom:none}.watch-table tbody tr>td[colspan]{grid-column:1/-1}.watch-table tbody tr>td.title-col{grid-column:1/-1;grid-row:1}.watch-table tbody tr>td.title-col .watch-title{font-size:.92rem}.watch-table tbody tr>td.title-col .link-spread{display:none}.watch-table tbody tr>td.last-checked{grid-column:1/-1;grid-row:2}.watch-table tbody tr>td.last-changed{grid-column:1/-1;grid-row:3}.watch-table tbody tr>td.checkbox-uuid{grid-column:1;grid-row:4}.watch-table tbody tr>td.buttons{grid-column:2;grid-row:4;display:flex;align-items:center;justify-content:flex-start}.watch-table tbody tr>td.watch-controls{grid-column:3;grid-row:4;display:grid;place-items:center}.watch-table tbody tr>td.watch-controls a img{padding:10px}.pure-table td{padding:3px !important}}ul#conditions_match_logic{list-style:none}ul#conditions_match_logic input,ul#conditions_match_logic label,ul#conditions_match_logic li{display:inline-block}ul#conditions_match_logic li{padding-right:1em}.fieldlist_formfields{width:100%;background-color:var(--color-background, #fff);border-radius:4px;border:1px solid var(--color-border-table-cell, #cbcbcb)}.fieldlist_formfields .fieldlist-header{display:flex;background-color:var(--color-background-table-thead, #e0e0e0);font-weight:bold;border-bottom:1px solid var(--color-border-table-cell, #cbcbcb)}.fieldlist_formfields .fieldlist-header-cell{flex:1;padding:.5em 1em;text-align:left}.fieldlist_formfields .fieldlist-header-cell:last-child{flex:0 0 120px}.fieldlist_formfields .fieldlist-body{display:flex;flex-direction:column}.fieldlist_formfields .fieldlist-row{display:flex;border-bottom:1px solid var(--color-border-table-cell, #cbcbcb)}.fieldlist_formfields .fieldlist-row:last-child{border-bottom:none}.fieldlist_formfields .fieldlist-row:nth-child(2n-1){background-color:var(--color-table-stripe, #f2f2f2)}.fieldlist_formfields .fieldlist-row.error-row{background-color:var(--color-error-input, #ffdddd)}.fieldlist_formfields .fieldlist-cell{flex:1;padding:.5em 1em;display:flex;flex-direction:column;justify-content:center}.fieldlist_formfields .fieldlist-cell input,.fieldlist_formfields .fieldlist-cell select{width:100%}.fieldlist_formfields .fieldlist-cell.fieldlist-actions{flex:0 0 120px;display:flex;flex-direction:row;align-items:center;gap:4px}.fieldlist_formfields ul.errors{margin-top:.5em;margin-bottom:0;padding:.5em;background-color:var(--color-error-background-snapshot-age, #ffdddd);border-radius:4px;list-style-position:inside}@media only screen and (max-width: 760px){.fieldlist_formfields .fieldlist-header,.fieldlist_formfields .fieldlist-row{flex-direction:column}.fieldlist_formfields .fieldlist-header-cell{display:none}.fieldlist_formfields .fieldlist-row{padding:.5em 0;border-bottom:2px solid var(--color-border-table-cell, #cbcbcb)}.fieldlist_formfields .fieldlist-cell{padding:.25em .5em}.fieldlist_formfields .fieldlist-cell.fieldlist-actions{flex:1;justify-content:flex-start;padding-top:.5em}.fieldlist_formfields .fieldlist-cell:not(:last-child){margin-bottom:.5em}.fieldlist_formfields .fieldlist-cell::before{content:attr(data-label);font-weight:bold;margin-bottom:.25em}}.fieldlist_formfields .addRuleRow,.fieldlist_formfields .removeRuleRow,.fieldlist_formfields .verifyRuleRow{cursor:pointer;border:none;padding:4px 8px;border-radius:3px;font-weight:bold;background-color:#aaa;color:var(--color-foreground-text, #fff)}.fieldlist_formfields .addRuleRow:hover,.fieldlist_formfields .removeRuleRow:hover,.fieldlist_formfields .verifyRuleRow:hover{background-color:#999}.watch-table.favicon-not-enabled tr .favicon{display:none}.watch-table tr td.inline.title-col .flex-wrapper{display:flex;align-items:center;gap:4px}.watch-table td,.watch-table th{vertical-align:middle}.watch-table tr.has-favicon.unviewed img.favicon{opacity:1 !important}.watch-table .status-icons{white-space:nowrap;display:flex;align-items:center;gap:4px}.watch-table .status-icons>*{vertical-align:middle}.title-col{padding:10px}.title-wrapper{display:flex;align-items:center;gap:10px}.title-col-inner{display:inline-block;vertical-align:middle}.watch-table img.favicon{vertical-align:middle;max-width:25px;max-height:25px;height:25px;padding-right:4px}body.checking-now #checking-now-fixed-tab{display:block !important}#checking-now-fixed-tab{background:#ccc;border-radius:5px;bottom:0;color:var(--color-text);display:none;font-size:.8rem;left:0;padding:5px;position:fixed}#selector-wrapper{height:100%;text-align:center;max-height:70vh;overflow-y:scroll;position:relative}#selector-wrapper>img{position:absolute;z-index:4;max-width:100%}#selector-wrapper>canvas{position:relative;z-index:5;max-width:100%}#selector-wrapper>canvas:hover{cursor:pointer}#selector-current-xpath{font-size:80%}.ternary-radio-group{display:flex;gap:0;border:1px solid var(--color-grey-750);border-radius:4px;overflow:hidden;width:fit-content;background:var(--color-background)}.ternary-radio-group .ternary-radio-option{position:relative;cursor:pointer;margin:0;display:flex;align-items:center}.ternary-radio-group .ternary-radio-option input[type=radio]{position:absolute;opacity:0;width:0;height:0}.ternary-radio-group .ternary-radio-option .ternary-radio-label{padding:8px 16px;background:var(--color-grey-900);border:none;border-right:1px solid var(--color-grey-750);font-size:13px;font-weight:500;color:var(--color-text);transition:all .2s ease;cursor:pointer;display:block;text-align:center}.ternary-radio-group .ternary-radio-option:last-child .ternary-radio-label{border-right:none}.ternary-radio-group .ternary-radio-option input:checked+.ternary-radio-label{background:var(--color-link);color:var(--color-text-button);font-weight:600}.ternary-radio-group .ternary-radio-option input:checked+.ternary-radio-label.ternary-default{background:var(--color-grey-600);color:var(--color-text-button)}.ternary-radio-group .ternary-radio-option input:checked+.ternary-radio-label:hover{background:#1a7bc4}.ternary-radio-group .ternary-radio-option input:checked+.ternary-radio-label:hover.ternary-default{background:var(--color-grey-500)}.ternary-radio-group .ternary-radio-option:hover .ternary-radio-label{background:var(--color-grey-800)}@media(max-width: 480px){.ternary-radio-group{width:100%}.ternary-radio-group .ternary-radio-label{flex:1;min-width:auto}}input[type=radio].pure-radio:checked+label,input[type=radio].pure-radio:checked{background:var(--color-link);color:var(--color-text-button)}html[data-darkmode=true] .ternary-radio-group .ternary-radio-option .ternary-radio-label{background:var(--color-grey-350)}html[data-darkmode=true] .ternary-radio-group .ternary-radio-option:hover .ternary-radio-label{background:var(--color-grey-400)}html[data-darkmode=true] .ternary-radio-group .ternary-radio-option input:checked+.ternary-radio-label{background:var(--color-link);color:var(--color-text-button)}html[data-darkmode=true] .ternary-radio-group .ternary-radio-option input:checked+.ternary-radio-label.ternary-default{background:var(--color-grey-600)}html[data-darkmode=true] .ternary-radio-group .ternary-radio-option input:checked+.ternary-radio-label:hover{background:#1a7bc4}html[data-darkmode=true] .ternary-radio-group .ternary-radio-option input:checked+.ternary-radio-label:hover.ternary-default{background:var(--color-grey-500)}body.processor-image_ssim_diff #edit-text-filter .text-filtering{display:none}body.processor-image_ssim_diff #conditions-tab{display:none}.modal-dialog{border:none;border-radius:10px;padding:0;background:var(--color-background);color:var(--color-text);box-shadow:0 5px 20px rgba(0,0,0,.3);max-width:500px;width:90%}.modal-dialog::backdrop{background:rgba(0,0,0,.6);backdrop-filter:blur(3px);animation:fadeIn .2s ease-out}.modal-dialog[open]{animation:slideIn .25s ease-out}.modal-dialog .modal-header{padding:1.5rem;border-bottom:1px solid var(--color-border-table-cell);display:flex;align-items:center;gap:1rem}.modal-dialog .modal-header .modal-icon{font-size:2rem;line-height:1;flex-shrink:0}.modal-dialog .modal-header .modal-icon.warning{color:var(--color-warning)}.modal-dialog .modal-header .modal-icon.danger{color:var(--color-background-button-error)}.modal-dialog .modal-header .modal-icon.info{color:var(--color-background-button-primary)}.modal-dialog .modal-header .modal-title{font-size:1.3rem;font-weight:bold;margin:0;color:var(--color-text)}.modal-dialog .modal-body{padding:1.5rem;line-height:1.6}.modal-dialog .modal-body p{margin:0 0 1rem 0}.modal-dialog .modal-body p:last-child{margin-bottom:0}.modal-dialog .modal-body strong{color:var(--color-text);font-weight:600}.modal-dialog .modal-footer{padding:1rem 1.5rem;border-top:1px solid var(--color-border-table-cell);display:flex;gap:.75rem;justify-content:flex-end;background:var(--color-grey-900)}.modal-dialog .modal-footer button{padding:.6rem 1.5rem;border:none;border-radius:4px;cursor:pointer;font-weight:500;transition:all .2s ease;font-size:.95rem}.modal-dialog .modal-footer button:hover{transform:translateY(-1px);box-shadow:0 2px 8px rgba(0,0,0,.15)}.modal-dialog .modal-footer button:active{transform:translateY(0)}.modal-dialog .modal-footer button.modal-btn-cancel{background:var(--color-background-button-cancel);color:var(--color-grey-200)}.modal-dialog .modal-footer button.modal-btn-cancel:hover{background:var(--color-grey-700)}.modal-dialog .modal-footer button.modal-btn-confirm{background:var(--color-background-button-primary);color:var(--color-white)}.modal-dialog .modal-footer button.modal-btn-confirm:hover{opacity:.9}.modal-dialog .modal-footer button.modal-btn-danger{background:var(--color-background-button-error);color:var(--color-white)}.modal-dialog .modal-footer button.modal-btn-danger:hover{background:var(--color-dark-red)}.modal-dialog .modal-footer button.modal-btn-warning{background:var(--color-background-button-warning);color:var(--color-white)}.modal-dialog .modal-footer button.modal-btn-warning:hover{opacity:.9}html[data-darkmode=true] .modal-dialog{box-shadow:0 5px 30px rgba(0,0,0,.7)}html[data-darkmode=true] .modal-dialog .modal-footer{background:var(--color-grey-200)}@keyframes fadeIn{from{opacity:0}to{opacity:1}}@keyframes slideIn{from{opacity:0;transform:translateY(-20px) scale(0.95)}to{opacity:1;transform:translateY(0) scale(1)}}@media only screen and (max-width: 760px){.modal-dialog{width:95%;max-width:none}.modal-dialog .modal-header{padding:1rem}.modal-dialog .modal-header .modal-title{font-size:1.1rem}.modal-dialog .modal-body{padding:1rem;font-size:.95rem}.modal-dialog .modal-footer{padding:.75rem 1rem;flex-wrap:wrap}.modal-dialog .modal-footer button{flex:1;min-width:120px}}#language-selector-flag{display:inline-block;width:1.2em;height:1.2em;vertical-align:middle;border-radius:50%;overflow:hidden;opacity:.6}#language-selector-flag:hover{opacity:1}.language-list{display:flex;flex-direction:column;gap:.5rem;padding:.5rem 0}.language-option{display:flex;align-items:center;gap:1rem;padding:.25rem;border-radius:4px;transition:background-color .2s ease;text-decoration:none;color:var(--color-text);border:1px solid rgba(0,0,0,0)}.language-option:hover{background-color:var(--color-background-menu-link-hover);border-color:var(--color-border-table-cell)}.language-option.active{background-color:var(--color-link);color:var(--color-text-button);font-weight:600}.language-option .flag{font-size:1.5rem;flex-shrink:0}.language-option .language-name{flex-grow:1;font-size:1rem}#language-modal .language-list .lang-option{display:inline-block;width:1.5em;height:1.5em;vertical-align:middle;margin-right:.5em;border-radius:50%;overflow:hidden}.content-wrapper{display:flex;gap:0;width:100%;max-width:100%;position:relative}@media only screen and (max-width: 900px){.content-wrapper{flex-direction:column}}.action-sidebar{position:sticky;top:100px;flex-shrink:0;width:80px;height:fit-content;background:rgba(0,0,0,0);padding:1.5rem 0;display:flex;flex-direction:column;gap:.5rem;align-items:center;z-index:0}@media only screen and (max-width: 900px){.action-sidebar{position:relative;top:0;width:100%;flex-direction:row;justify-content:space-around;padding:0;overflow-x:auto}}.action-sidebar-item{position:relative;display:flex;flex-direction:column;align-items:center;justify-content:center;gap:.35rem;padding:.75rem .5rem;min-width:64px;text-decoration:none;opacity:.8;transition:opacity .2s ease}.action-sidebar-item:hover{opacity:1}.action-sidebar-item.active{opacity:1}.action-sidebar-item.active .action-icon{stroke:#fff;stroke-width:2.5}.action-sidebar-item.active .action-label{color:#fff;font-weight:700}.action-icon{width:28px;height:28px;stroke:#fff;stroke-width:2;fill:none;stroke-linecap:round;stroke-linejoin:round;transition:stroke .2s ease}.action-label{font-size:.65rem;font-weight:500;text-align:center;line-height:1.1;letter-spacing:.02em;text-transform:uppercase;color:#fff;transition:color .2s ease;max-width:60px;word-wrap:break-word}.content-main{flex:0 1 auto;width:100%;min-width:0;padding:0;display:flex;flex-direction:column;align-items:center}.hamburger-menu{display:none;background:rgba(0,0,0,0);border:none;cursor:pointer;padding:.5rem;z-index:10001;position:relative}@media only screen and (max-width: 980px){.hamburger-menu{display:flex;flex-direction:column;justify-content:center;align-items:center}}.hamburger-icon{width:24px;height:20px;position:relative;display:flex;flex-direction:column;justify-content:space-between}.hamburger-icon span{display:block;height:3px;width:100%;background:var(--color-text);border-radius:2px;transition:all .3s cubic-bezier(0.68, -0.55, 0.265, 1.55);transform-origin:center}.hamburger-menu.active .hamburger-icon span:nth-child(1){transform:translateY(8.5px) rotate(45deg)}.hamburger-menu.active .hamburger-icon span:nth-child(2){opacity:0;transform:translateX(-10px)}.hamburger-menu.active .hamburger-icon span:nth-child(3){transform:translateY(-8.5px) rotate(-45deg)}.mobile-menu-overlay{display:none;position:fixed;top:0;left:0;right:0;bottom:0;background:rgba(0,0,0,.5);z-index:9999;opacity:0;transition:opacity .3s ease}.mobile-menu-overlay.active{display:block;opacity:1}.mobile-menu-drawer{position:fixed;top:0;right:-280px;width:280px;height:100%;background:var(--color-background);opacity:1;box-shadow:-2px 0 8px rgba(0,0,0,.15);z-index:10000;transition:right .3s cubic-bezier(0.68, -0.55, 0.265, 1.55);overflow-y:auto;padding-top:60px}.mobile-menu-drawer.active{right:0}.mobile-menu-drawer .mobile-menu-items{list-style:none;padding:1rem 0;margin:0}.mobile-menu-drawer .mobile-menu-items li{border-bottom:1px solid var(--color-border-table-cell)}.mobile-menu-drawer .mobile-menu-items li>*{display:block;padding:1rem 1.5rem;color:var(--color-text);text-decoration:none;font-weight:500;transition:background .2s ease}.mobile-menu-drawer .mobile-menu-items li>*:hover{background:var(--color-background-menu-link-hover)}.mobile-menu-drawer .mobile-menu-items li#menu-pause,.mobile-menu-drawer .mobile-menu-items li#menu-mute{display:none}.logo-cdio{font-weight:bold;font-size:1.1rem}.logo-cdio .logo-cd{color:var(--color-grey-500)}.logo-cdio .logo-io{color:var(--color-text)}.menu-always-visible{display:flex;align-items:center;gap:.5rem;margin-left:auto}@media only screen and (max-width: 980px){#top-right-menu .menu-collapsible{display:none !important}.pure-menu-horizontal{overflow-x:visible !important}#nav-menu{overflow-x:visible !important}}@media only screen and (min-width: 1025px){.hamburger-menu,.mobile-menu-drawer,.mobile-menu-overlay{display:none !important}}html[data-darkmode=true] .mobile-menu-drawer{box-shadow:-2px 0 8px rgba(0,0,0,.4)}#search-modal .modal-body{padding:2rem 1.5rem}#search-modal .modal-body .pure-control-group{padding-bottom:0}#search-modal .modal-body .pure-control-group label{display:block;margin-bottom:.5rem;font-size:.9rem;font-weight:600;color:var(--color-text)}#search-modal .modal-body .pure-control-group #search-modal-input{width:100%;max-width:100%;box-sizing:border-box;padding:.6rem .8rem;font-size:1rem;border:1px solid var(--color-border-input);border-radius:4px;background-color:var(--color-background-input);color:var(--color-text-input);box-shadow:inset 0 1px 3px var(--color-shadow-input);transition:border-color .2s ease,box-shadow .2s ease}#search-modal .modal-body .pure-control-group #search-modal-input:focus{outline:none;border-color:var(--color-link);box-shadow:0 0 0 3px rgba(27,152,248,.1)}#search-modal .modal-body .pure-control-group #search-modal-input::placeholder{color:var(--color-text-input-placeholder);opacity:.7}html[data-darkmode=true] #search-modal #search-modal-input:focus{box-shadow:0 0 0 3px rgba(89,189,251,.15)}.action-sidebar-item{position:relative}.action-sidebar-item .notification-bubble{position:absolute;top:8px;left:8px;min-width:18px;height:18px;background:#f44;color:#fff;font-size:10px;font-weight:700;line-height:18px;text-align:center;border-radius:9px;padding:0 2px;box-shadow:0 2px 4px rgba(0,0,0,.3);pointer-events:none;transition:all .2s ease;display:none}.action-sidebar-item .notification-bubble.red-bubble{background:#f44}.action-sidebar-item .notification-bubble.blue-bubble{background:#4a9eff;color:#fff}.action-sidebar-item .notification-bubble.visible{display:block}.action-sidebar-item .notification-bubble.pulse{animation:bubblePulse .4s ease-out}.action-sidebar-item .notification-bubble.large-number{font-size:8px;min-width:20px;height:20px;line-height:20px;border-radius:10px}@keyframes bubblePulse{0%{transform:scale(1)}50%{transform:scale(1.3)}100%{transform:scale(1)}}html[data-darkmode=true] .notification-bubble{box-shadow:0 2px 6px rgba(0,0,0,.6)}.toast-container{position:fixed;display:flex;flex-direction:column;gap:.75rem;pointer-events:none;z-index:10000}.toast-container.toast-top-right{top:20px;right:20px}.toast-container.toast-top-center{top:100px;left:50%;transform:translateX(-50%)}.toast-container.toast-top-left{top:20px;left:20px}.toast-container.toast-bottom-right{bottom:20px;right:20px}.toast-container.toast-bottom-center{bottom:20px;left:50%;transform:translateX(-50%)}.toast-container.toast-bottom-left{bottom:20px;left:20px}.toast{position:relative;display:flex;align-items:center;gap:.75rem;min-width:300px;max-width:500px;padding:1rem 1.25rem;background:var(--color-background);border-radius:8px;box-shadow:0 4px 12px rgba(0,0,0,.15),0 0 0 1px rgba(0,0,0,.05);pointer-events:auto;overflow:hidden;opacity:0;transform:translateY(-50px);transition:all .3s cubic-bezier(0.68, -0.55, 0.265, 1.55);font-family:inherit}.toast.toast-show{opacity:1;transform:translateY(0)}.toast.toast-hide{opacity:0;transform:translateY(-50px) scale(0.95)}.toast.toast-success{border-left:4px solid #10b981}.toast.toast-success .toast-icon{color:#10b981}.toast.toast-error{border-left:4px solid #ef4444}.toast.toast-error .toast-icon{color:#ef4444}.toast.toast-warning{border-left:4px solid #f59e0b}.toast.toast-warning .toast-icon{color:#f59e0b}.toast.toast-info{border-left:4px solid #3b82f6}.toast.toast-info .toast-icon{color:#3b82f6}.toast.toast-default{border-left:4px solid var(--color-grey-500)}.toast-icon{flex-shrink:0;width:24px;height:24px}.toast-icon svg{width:100%;height:100%}.toast-message{flex:1;font-size:.875rem;line-height:1.5;color:var(--color-text);word-break:break-word;font-family:inherit}.toast-close{flex-shrink:0;width:24px;height:24px;display:flex;align-items:center;justify-content:center;background:rgba(0,0,0,0);border:none;border-radius:4px;color:var(--color-grey-500);font-size:1.5rem;line-height:1;cursor:pointer;transition:all .2s ease;padding:0;margin-left:.25rem}.toast-close:hover{background:var(--color-grey-800);color:var(--color-text)}.toast-close:active{transform:scale(0.95)}.toast-progress{position:absolute;bottom:0;left:0;right:0;height:3px;background:currentColor;opacity:.3;transform-origin:left;transition:transform linear}html[data-darkmode=true] .toast{background:var(--color-grey-300);box-shadow:0 4px 12px rgba(0,0,0,.4),0 0 0 1px hsla(0,0%,100%,.05)}html[data-darkmode=true] .toast-close:hover{background:var(--color-grey-400)}@media only screen and (max-width: 768px){.toast-container{left:50% !important;right:auto !important;top:80px !important;transform:translateX(-50%) !important;align-items:center}.toast-container.toast-bottom-right,.toast-container.toast-bottom-center,.toast-container.toast-bottom-left{top:auto !important;bottom:80px !important}.toast{min-width:auto;max-width:none;width:80vw;transform:translateY(-100px)}.toast.toast-show{transform:translateY(0)}.toast.toast-hide{transform:translateY(-100px) scale(0.95)}}@media(prefers-reduced-motion: reduce){.toast{transition:opacity .2s ease;transform:none !important}.toast.toast-show{opacity:1}.toast.toast-hide{opacity:0}}.login-form{min-height:52vh;display:flex;align-items:center;justify-content:center;padding:2rem 1rem}.login-form .inner{background:var(--color-background);border-radius:16px;box-shadow:0 10px 40px rgba(0,0,0,.08),0 2px 8px rgba(0,0,0,.04);padding:3rem 2.5rem;width:100%;max-width:420px;position:relative;overflow:hidden;transition:transform .3s ease,box-shadow .3s ease}.login-form .inner:hover{box-shadow:0 15px 50px rgba(0,0,0,.12),0 5px 15px rgba(0,0,0,.06)}.login-form form{margin:0}.login-form fieldset{border:none;padding:0;margin:0}.login-form .pure-control-group{margin-bottom:1.75rem}.login-form .pure-control-group:last-of-type{margin-bottom:0;margin-top:2rem}.login-form label{display:block;margin-bottom:.5rem;font-weight:600;font-size:.9rem;color:var(--color-text);letter-spacing:.01em}.login-form input[type=password]{width:100%;padding:.875rem 1rem;border:2px solid var(--color-grey-800);border-radius:8px;font-size:1rem;background:var(--color-background-input);color:var(--color-text-input);transition:all .2s ease;box-sizing:border-box}.login-form input[type=password]:focus{outline:none;border-color:var(--color-link);box-shadow:0 0 0 3px rgba(27,152,248,.1);transform:translateY(-1px)}.login-form input[type=password]::placeholder{color:var(--color-text-input-placeholder)}.login-form button[type=submit]{width:100%;padding:.875rem 1.5rem;font-size:1rem;font-weight:600;border-radius:8px;border:none;background:var(--color-background-button-primary);color:var(--color-text-button);cursor:pointer;transition:all .2s ease;box-shadow:0 2px 8px rgba(27,152,248,.2)}.login-form button[type=submit]:hover{box-shadow:0 4px 12px rgba(27,152,248,.3);background:#06c}.login-form button[type=submit]:active{transform:translateY(0);box-shadow:0 2px 4px rgba(27,152,248,.2)}.content-main>ul.messages{position:fixed;top:120px;left:50%;transform:translateX(-50%);list-style:none;padding:0;margin:0;z-index:1000;min-width:300px;max-width:500px}.content-main>ul.messages li{padding:1rem 1.25rem;border-radius:8px;font-size:.95rem;line-height:1.5;font-weight:500;box-shadow:0 4px 12px rgba(0,0,0,.15);animation:slideDown .3s ease-out;border:2px solid rgba(0,0,0,0)}.content-main>ul.messages li.error{background:#fee;border:2px solid #ef4444;color:#991b1b;font-weight:600}.content-main>ul.messages li.success{background:#f0fdf4;border:2px solid #10b981;color:#166534}.content-main>ul.messages li.info,.content-main>ul.messages li.message{background:#eff6ff;border:2px solid #3b82f6;color:#1e40af}@keyframes slideDown{from{opacity:0;transform:translateY(-20px)}to{opacity:1;transform:translateY(0)}}html[data-darkmode=true] .login-form .inner{box-shadow:0 10px 40px rgba(0,0,0,.4),0 2px 8px rgba(0,0,0,.2)}html[data-darkmode=true] .login-form .inner:hover{box-shadow:0 15px 50px rgba(0,0,0,.5),0 5px 15px rgba(0,0,0,.3)}html[data-darkmode=true] .login-form input[type=password]{border-color:var(--color-grey-400)}html[data-darkmode=true] .login-form input[type=password]:focus{border-color:var(--color-link)}html[data-darkmode=true] .content-main>ul.messages li{box-shadow:0 4px 12px rgba(0,0,0,.4)}html[data-darkmode=true] .content-main>ul.messages li.error{background:#4a1d1d;border-color:#ef4444;color:#fca5a5}html[data-darkmode=true] .content-main>ul.messages li.success{background:#1a3a2a;border-color:#10b981;color:#86efac}html[data-darkmode=true] .content-main>ul.messages li.info,html[data-darkmode=true] .content-main>ul.messages li.message{background:#1e3a5f;border-color:#3b82f6;color:#93c5fd}@media only screen and (max-width: 768px){.login-form{min-height:auto;padding:1rem .5rem;padding-top:5rem}.login-form .inner{padding:2rem 1.5rem;border-radius:12px}.content-main>ul.messages{top:70px;left:10px;right:10px;transform:none;min-width:auto}}body.wrapped-tabs .tabs ul{grid-template-columns:repeat(auto-fill, minmax(var(--tab-width, 180px), 1fr));grid-auto-flow:row;grid-auto-columns:unset;gap:0;column-gap:5px}body.wrapped-tabs .tabs ul li{border-radius:0}.tabs ul{margin:0px;padding:0px;display:grid;grid-auto-flow:column;grid-auto-columns:max-content;gap:5px;list-style:none}.tabs ul li{white-space:nowrap;color:var(--color-text-tab);border-top-left-radius:5px;border-top-right-radius:5px;background-color:var(--color-background-tab)}.tabs ul li:not(.active):hover{background-color:var(--color-background-tab-hover)}.tabs ul li.active,.tabs ul li :target{background-color:var(--color-background)}.tabs ul li.active a,.tabs ul li :target a{color:var(--color-text-tab-active);font-weight:bold}.tabs ul li a{display:block;padding:.7em;color:var(--color-text-tab)}body,.pure-table,.pure-table thead,.pure-table td,.pure-table th,.pure-form input,.pure-form textarea,.pure-form select,.edit-form .inner,.pure-menu-horizontal,footer,.sticky-tab,#diff-jump,.button-tag,#new-watch-form,#new-watch-form input:not(.pure-button),code,.messages li,#checkbox-operations,.inline-warning,a,.watch-controls img{transition:color .4s ease,background-color .4s ease,background .4s ease,border-color .4s ease,box-shadow .4s ease}body{color:var(--color-text);background:var(--color-background-page);font-family:Helvetica Neue,Helvetica,Lucida Grande,Arial,Ubuntu,Cantarell,Fira Sans,sans-serif}.visually-hidden{clip:rect(0 0 0 0);clip-path:inset(50%);height:1px;overflow:hidden;position:absolute;white-space:nowrap;width:1px}.status-icon{display:inline-block;height:1rem;vertical-align:middle}.pure-table-even{background:var(--color-background)}a{text-decoration:none;color:var(--color-link)}a.github-link{color:var(--color-icon-github);margin:0 1rem 0 .5rem}a.github-link svg{fill:currentColor}a.github-link:hover{color:var(--color-icon-github-hover)}#search-result-info{color:#fff}button.toggle-button{vertical-align:middle;background:rgba(0,0,0,0);border:none;cursor:pointer;color:var(--color-icon-github)}button.toggle-button:hover{color:var(--color-icon-github-hover)}button.toggle-button svg{fill:currentColor}button.toggle-button .icon-light{display:block}.pure-menu-horizontal{background:var(--color-background);padding:5px;display:flex;justify-content:space-between;align-items:center}#pure-menu-horizontal-spinner{height:3px;background:linear-gradient(-75deg, #ff6000, #ff8f00, #ffdd00, #ed0000);background-size:400% 400%;width:100%;animation:gradient 200s ease infinite}body.spinner-active #pure-menu-horizontal-spinner{animation:gradient 1s ease infinite}@keyframes gradient{0%{background-position:0% 50%}50%{background-position:100% 50%}100%{background-position:0% 50%}}.pure-menu-heading{color:var(--color-text-menu-heading)}.pure-menu-link{color:var(--color-text-menu-link)}.pure-menu-link:hover{background-color:var(--color-background-menu-link-hover);color:var(--color-text-menu-link-hover)}.tab-pane-inner{scroll-margin-top:200px}section.content{padding-bottom:1em;flex-direction:column;display:flex;align-items:center;justify-content:center}@media only screen and (max-width: 980px){section.content{padding-top:80px}}@media only screen and (min-width: 980px){section.content{padding-top:100px}}code{background:var(--color-background-code);color:var(--color-text)}.inline-tag,.restock-label,.tracking-ldjson-price-data,.watch-tag-list,.processor-badge{white-space:nowrap;border-radius:5px;padding:2px 5px;margin-right:4px;line-height:1.2rem}.processor-badge{font-weight:900}.watch-tag-list{color:var(--color-white);background:var(--color-text-watch-tag-list);text-decoration:none}.watch-tag-list:hover{text-decoration:none;opacity:.8;cursor:pointer}.watch-tag-list:visited{color:var(--color-white)}@media(min-width: 768px){.box{margin:0 1em !important}}.box{max-width:100%;margin:0 .3em;flex-direction:column;display:flex;justify-content:center}body:after{content:\"\";background:linear-gradient(130deg, var(--color-background-gradient-first), var(--color-background-gradient-second) 41.07%, var(--color-background-gradient-third) 84.05%)}body:after,body:before{display:block;height:650px;position:absolute;top:0;left:0;width:100%;z-index:-1}body::after{opacity:.91}body::before{content:\"\"}body:after,body:before{-webkit-clip-path:polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%);clip-path:polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%)}.button-small{font-size:85%}.button-xsmall{font-size:70%}.fetch-error{padding-top:1em;font-size:80%;max-width:400px;display:block}.pure-button-primary,a.pure-button-primary,.pure-button-selected,a.pure-button-selected{background-color:var(--color-background-button-primary)}.button-secondary{color:var(--color-text-button);border-radius:4px;text-shadow:0 1px 1px rgba(0,0,0,.2)}.button-success{background:var(--color-background-button-success)}.button-tag{background:var(--color-background-button-tag);color:var(--color-text-button);font-size:65%;border-bottom-left-radius:initial;border-bottom-right-radius:initial;margin-right:4px}.button-tag.active{background:var(--color-background-button-tag-active);font-weight:bold}.button-error{background:var(--color-background-button-error);color:var(--color-text-button-error)}.button-warning{background:var(--color-background-button-warning);color:var(--color-text-button-warning)}.button-secondary{background:var(--color-background-button-secondary)}.button-cancel{background:var(--color-background-button-cancel)}.messages li{list-style:none;padding:1em;border-radius:10px;color:var(--color-text-messages);font-weight:bold}.messages li.message{background:var(--color-background-messages-message)}.messages li.error{background:var(--color-background-messages-error)}.messages li.notice{background:var(--color-background-messages-notice)}.messages.with-share-link>*:hover{cursor:pointer}.notifications-wrapper{padding-top:.5rem}.notifications-wrapper #notification-test-log{margin-top:1rem;padding:1rem;white-space:pre-wrap;word-break:break-word;overflow-wrap:break-word;max-width:100%;box-sizing:border-box;max-height:12rem;overflow-y:scroll;border:1px solid var(--color-border-notification);border-radius:5px}label:hover{cursor:pointer}.grey-form-border{border:1px solid var(--color-border-notification);padding:.5rem;border-radius:5px}#notification-error-log{border:1px solid var(--color-border-notification);padding:1rem;border-radius:5px;overflow-wrap:break-word}#token-table.pure-table td,#token-table.pure-table th{font-size:80%}.pure-form input[type=text].transparent-field{background-color:var(--color-background-new-watch-input-transparent) !important;color:var(--color-white) !important;border:1px solid hsla(0,0%,100%,.2) !important;box-shadow:none !important;-webkit-box-shadow:none !important}.pure-form input[type=text].transparent-field::placeholder{opacity:.5;color:hsla(0,0%,100%,.7);font-weight:lighter}#new-watch-form{background:var(--color-background-new-watch-form);padding:1em;border-radius:10px;margin-bottom:1em;max-width:100%}#new-watch-form #url::placeholder{font-weight:bold}#new-watch-form input{display:inline-block;margin-bottom:5px}#new-watch-form input:not(.pure-button){background-color:var(--color-background-new-watch-input);color:var(--color-text-new-watch-input)}#new-watch-form .label{display:none}#new-watch-form legend{color:var(--color-text-legend);font-weight:bold}@media only screen and (min-width: 760px){#new-watch-form #watch-add-wrapper-zone{display:flex;gap:.3rem;flex-direction:row;min-width:70vw}}#new-watch-form #watch-add-wrapper-zone>span{flex-grow:0}#new-watch-form #watch-add-wrapper-zone>span input{width:100%;padding-right:1em}#new-watch-form #watch-add-wrapper-zone>span:first-child{flex-grow:1}@media only screen and (max-width: 760px){#new-watch-form #watch-add-wrapper-zone #url{width:100%}}#new-watch-form #watch-group-tag{font-size:.9rem;padding:.3rem;display:flex;align-items:center;gap:.5rem;color:var(--color-white)}#new-watch-form #watch-group-tag label,#new-watch-form #watch-group-tag input{margin:0}#new-watch-form #watch-group-tag input{flex:1}#diff-col{padding-left:40px}#diff-jump{position:fixed;left:0px;top:120px;background:var(--color-background);padding:10px;border-top-right-radius:5px;border-bottom-right-radius:5px;box-shadow:1px 1px 4px var(--color-shadow-jump)}#diff-jump a{color:var(--color-link);cursor:pointer;-moz-user-select:none;-webkit-user-select:none;-ms-user-select:none;user-select:none;-o-user-select:none}footer{padding:10px;background:var(--color-background);color:var(--color-text-footer);text-align:center}#feed-icon{vertical-align:middle}.sticky-tab{position:absolute;top:60px;font-size:65%;background:var(--color-background);padding:10px}@media only screen and (max-width: 980px){.sticky-tab{display:none}}.sticky-tab#left-sticky{left:0;position:fixed;border-top-right-radius:5px;border-bottom-right-radius:5px;box-shadow:1px 1px 4px var(--color-shadow-jump)}.sticky-tab#right-sticky{right:0px}.sticky-tab#hosted-sticky{right:0px;top:100px;font-weight:bold}#new-version-text a{color:var(--color-link-new-version)}.watch-controls{color:#f8321b}.watch-controls .state-on img{opacity:.8}.watch-controls img{opacity:.2}.watch-controls img:hover{transition:opacity .3s;opacity:.8}.monospaced-textarea textarea{width:100%;font-family:monospace;white-space:pre;overflow-wrap:normal;overflow-x:auto}.pure-form fieldset{padding-top:0px}.pure-form fieldset ul{padding-bottom:0px;margin-bottom:0px}.pure-form .pure-control-group,.pure-form .pure-group,.pure-form .pure-controls{padding-bottom:1em}.pure-form .pure-control-group div,.pure-form .pure-group div,.pure-form .pure-controls div{margin:0px}.pure-form .pure-control-group .checkbox>*,.pure-form .pure-group .checkbox>*,.pure-form .pure-controls .checkbox>*{display:inline;vertical-align:middle}.pure-form .pure-control-group .checkbox>label,.pure-form .pure-group .checkbox>label,.pure-form .pure-controls .checkbox>label{padding-left:5px}.pure-form .pure-control-group legend,.pure-form .pure-group legend,.pure-form .pure-controls legend{color:var(--color-text-legend)}.pure-form .error input{background-color:var(--color-error-input)}.pure-form ul.errors{padding:.5em .6em;border:1px solid var(--color-error-list);border-radius:4px;vertical-align:middle;-webkit-box-sizing:border-box;box-sizing:border-box}.pure-form ul.errors li{margin-left:1em;color:var(--color-error-list)}.pure-form label{font-weight:bold}.pure-form textarea{width:100%}.pure-form .inline-radio ul{margin:0px;list-style:none}.pure-form .inline-radio ul li{display:flex;align-items:center;gap:1em}@media only screen and (max-width: 760px),(min-device-width: 768px)and (max-device-width: 980px){.edit-form{padding:.5em;margin:0}#nav-menu{overflow-x:scroll}}@media only screen and (max-width: 760px),(min-device-width: 768px)and (max-device-width: 980px){input[type=text]{width:100%}}.pure-table{border-color:var(--color-border-table-cell)}.pure-table thead{background-color:var(--color-background-table-thead);color:var(--color-text);border-bottom:1px solid var(--color-background-table-thead)}.pure-table td,.pure-table th{border-left-color:var(--color-border-table-cell)}.pure-table-striped tr:nth-child(2n-1) td{background-color:var(--color-table-stripe)}.pure-form input[type=color],.pure-form input[type=date],.pure-form input[type=datetime-local],.pure-form input[type=datetime],.pure-form input[type=email],.pure-form input[type=month],.pure-form input[type=number],.pure-form input[type=password],.pure-form input[type=search],.pure-form input[type=tel],.pure-form input[type=text],.pure-form input[type=time],.pure-form input[type=url],.pure-form input[type=week],.pure-form select,.pure-form textarea{border:var(--color-border-input);box-shadow:inset 0 1px 3px var(--color-shadow-input);background-color:var(--color-background-input);color:var(--color-text-input)}.pure-form input[type=color]:active,.pure-form input[type=date]:active,.pure-form input[type=datetime-local]:active,.pure-form input[type=datetime]:active,.pure-form input[type=email]:active,.pure-form input[type=month]:active,.pure-form input[type=number]:active,.pure-form input[type=password]:active,.pure-form input[type=search]:active,.pure-form input[type=tel]:active,.pure-form input[type=text]:active,.pure-form input[type=time]:active,.pure-form input[type=url]:active,.pure-form input[type=week]:active,.pure-form select:active,.pure-form textarea:active{background-color:var(--color-background-input)}input::placeholder,textarea::placeholder{color:var(--color-text-input-placeholder)}.m-d{min-width:100%}@media only screen and (min-width: 761px){.m-d{min-width:80%}}.pure-form-stacked>div:first-child{display:block}.tab-pane-inner{padding:0px}.tab-pane-inner:not(:target){display:none}.tab-pane-inner:target{display:block}.beta-logo{height:50px;right:-3px;top:-3px;position:absolute}#selector-header{padding-bottom:1em}body.full-width .edit-form{width:95%}.edit-form{min-width:70%;max-width:95%}.edit-form .box-wrap{position:relative}.edit-form .inner{background:var(--color-background);padding:20px}.edit-form #actions{display:block;background:var(--color-background)}.edit-form #actions .pure-control-group{display:flex;gap:.625em;flex-wrap:wrap}.edit-form .pure-form-message-inline{padding-left:0;color:var(--color-text-input-description)}.edit-form .pure-form-message-inline code{font-size:.875em}.border-fieldset{border:1px solid #ccc;padding:1rem;border-radius:5px;margin-bottom:1rem}.border-fieldset h3{margin-top:0}.border-fieldset fieldset:last-of-type{padding-bottom:0}.border-fieldset fieldset:last-of-type .pure-control-group{padding-bottom:0}ul{padding-left:1em;padding-top:0px;margin-top:4px}.time-check-widget tr{display:inline}.time-check-widget tr input[type=number]{width:5em}@media only screen and (max-width: 760px){.time-check-widget tbody{display:grid;grid-template-columns:auto 1fr auto 1fr;gap:.625em .3125em;align-items:center}.time-check-widget tr{display:contents}.time-check-widget tr th{text-align:right;padding-right:5px}.time-check-widget tr input[type=number]{width:100%;max-width:5em}}#webdriver_delay{width:5em}#api-key:hover{cursor:pointer}#api-key-copy{color:var(--color-api-key)}.button-green{background-color:var(--color-background-button-green)}.button-red{background-color:var(--color-background-button-red)}.noselect{-webkit-touch-callout:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none}#checkbox-operations{background:var(--color-background-checkbox-operations);padding:1em;border-radius:10px;margin-bottom:1em;display:none}#checkbox-operations button{margin-bottom:3px;margin-top:3px;display:inline-flex;align-items:center}.checkbox-uuid>*{vertical-align:middle}.inline-warning{border:1px solid var(--color-border-warning);padding:.5rem;border-radius:5px;color:var(--color-warning)}.inline-warning>span{display:inline-block;vertical-align:middle}.inline-warning img.inline-warning-icon{display:inline;height:26px;vertical-align:middle}.tracking-ldjson-price-data{background-color:var(--color-background-button-green);color:#000;opacity:.6}.ldjson-price-track-offer{font-weight:bold;font-style:italic}.ldjson-price-track-offer a.pure-button{border-radius:3px;padding:3px;background-color:var(--color-background-button-green)}.price-follow-tag-icon{display:inline-block;height:.8rem;vertical-align:middle}#quick-watch-processor-type ul#processor{color:#fff;padding-left:0px}#quick-watch-processor-type ul#processor li{list-style:none;font-size:.9rem;display:grid;grid-template-columns:auto 1fr;align-items:center;gap:.5rem;margin-bottom:.5rem}#quick-watch-processor-type label,#quick-watch-processor-type input{padding:0;margin:0}.restock-label.in-stock{background-color:var(--color-background-button-green);color:#fff}.restock-label.not-in-stock{background-color:var(--color-background-button-cancel);color:#777}.restock-label.error{background-color:var(--color-background-button-error);color:#fff;opacity:.7}.restock-label svg{vertical-align:middle}#chrome-extension-link{padding:9px;border:1px solid var(--color-grey-800);border-radius:10px;vertical-align:middle}#chrome-extension-link img{height:21px;padding:2px;vertical-align:middle}#realtime-conn-error{position:fixed;bottom:0;left:0;background:var(--color-warning);padding:10px;font-size:.8rem;color:#fff;opacity:.8}#bottom-horizontal-offscreen{position:fixed;bottom:0;left:0;right:0;width:100%;min-height:50px;max-height:50vh;background:hsla(0,0%,100%,.7215686275);border-top:1px solid var(--color-border-table-cell);padding:10px;box-shadow:0 -2px 10px rgba(0,0,0,.2);z-index:100;overflow-y:auto;transition:opacity .3s ease-in-out;scroll-margin-bottom:10px;display:flex;justify-content:center;align-items:center}ul#highlightSnippetActions{list-style:none}ul#highlightSnippetActions li{display:inline-block}\n"
  },
  {
    "path": "changedetectionio/store/__init__.py",
    "content": "import shutil\n\nfrom changedetectionio.strtobool import strtobool\n\nfrom changedetectionio.validate_url import is_safe_valid_url\n\nfrom flask import (\n    flash\n)\nfrom flask_babel import gettext\n\nfrom ..model import App, Watch\nfrom copy import deepcopy\nfrom os import path, unlink\nimport json\nimport os\nimport re\nimport secrets\nimport sys\nimport time\nimport uuid as uuid_builder\nfrom loguru import logger\nfrom blinker import signal\n\nfrom ..model.Tags import TagsDict\n\n# Try to import orjson for faster JSON serialization\ntry:\n    import orjson\n\n    HAS_ORJSON = True\nexcept ImportError:\n    HAS_ORJSON = False\n\nfrom ..processors import get_custom_watch_obj_for_processor\n\n# Import the base class and helpers\nfrom .file_saving_datastore import FileSavingDataStore, load_all_watches, load_all_tags, save_json_atomic\nfrom .updates import DatastoreUpdatesMixin\n\n# Because the server will run as a daemon and wont know the URL for notification links when firing off a notification\nBASE_URL_NOT_SET_TEXT = '(\"Base URL\" not set - see settings - notifications)'\n\ndictfilt = lambda x, y: dict([(i, x[i]) for i in x if i in set(y)])\n\n\n# Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods?\n# Open a github issue if you know something :)\n# https://stackoverflow.com/questions/6190468/how-to-trigger-function-on-value-change\nclass ChangeDetectionStore(DatastoreUpdatesMixin, FileSavingDataStore):\n    __version_check = True\n\n    def __init__(self, datastore_path=\"/datastore\", include_default_watches=True, version_tag=\"0.0.0\"):\n        # Initialize parent class\n        super().__init__()\n\n        # Should only be active for docker\n        # logging.basicConfig(filename='/dev/stdout', level=logging.INFO)\n        self.datastore_path = datastore_path\n        self.start_time = time.time()\n        self.save_version_copy_json_db(version_tag)\n        self.reload_state(datastore_path=datastore_path, include_default_watches=include_default_watches, version_tag=version_tag)\n\n    def save_version_copy_json_db(self, version_tag):\n        \"\"\"\n        Create version-tagged backup of changedetection.json.\n\n        This is called on version upgrades to preserve a backup in case\n        the new version has issues.\n        \"\"\"\n        import re\n\n        version_text = re.sub(r'\\D+', '-', version_tag)\n        db_path = os.path.join(self.datastore_path, \"changedetection.json\")\n        db_path_version_backup = os.path.join(self.datastore_path, f\"changedetection-{version_text}.json\")\n\n        if not os.path.isfile(db_path_version_backup) and os.path.isfile(db_path):\n            from shutil import copyfile\n            logger.info(f\"Backing up changedetection.json due to new version to '{db_path_version_backup}'.\")\n            copyfile(db_path, db_path_version_backup)\n\n    def _load_settings(self, filename=\"changedetection.json\"):\n        \"\"\"\n        Load settings from storage.\n\n        File backend implementation: reads from changedetection.json\n\n        Returns:\n            dict: Settings data loaded from storage\n        \"\"\"\n        changedetection_json = os.path.join(self.datastore_path, filename)\n\n        logger.info(f\"Loading settings from {changedetection_json}\")\n\n        if HAS_ORJSON:\n            with open(changedetection_json, 'rb') as f:\n                return orjson.loads(f.read())\n        else:\n            with open(changedetection_json, 'r', encoding='utf-8') as f:\n                return json.load(f)\n\n    def _apply_settings(self, settings_data):\n        \"\"\"\n        Apply loaded settings data to internal data structure.\n\n        Args:\n            settings_data: Dictionary loaded from changedetection.json\n        \"\"\"\n        # Apply top-level fields\n        if 'app_guid' in settings_data:\n            self.__data['app_guid'] = settings_data['app_guid']\n        if 'build_sha' in settings_data:\n            self.__data['build_sha'] = settings_data['build_sha']\n        if 'version_tag' in settings_data:\n            self.__data['version_tag'] = settings_data['version_tag']\n\n        # Apply settings sections\n        if 'settings' in settings_data:\n            if 'headers' in settings_data['settings']:\n                self.__data['settings']['headers'].update(settings_data['settings']['headers'])\n            if 'requests' in settings_data['settings']:\n                self.__data['settings']['requests'].update(settings_data['settings']['requests'])\n            if 'application' in settings_data['settings']:\n                self.__data['settings']['application'].update(settings_data['settings']['application'])\n\n                # Use our Tags dict with cleanup helpers etc\n                # @todo Same for Watches\n                existing_tags = settings_data.get('settings', {}).get('application', {}).get('tags') or {}\n                self.__data['settings']['application']['tags'] = TagsDict(existing_tags, datastore_path=self.datastore_path)\n\n        # More or less for the old format which had this data in the one url-watches.json\n        # cant hurt to leave it here,\n        if 'watching' in settings_data:\n            self.__data['watching'].update(settings_data['watching'])\n\n    def _rehydrate_tags(self):\n        \"\"\"Rehydrate tag entities from stored data into Tag objects with restock_diff processor.\"\"\"\n        from ..model import Tag\n\n        for uuid, tag in self.__data['settings']['application']['tags'].items():\n            # Force processor to restock_diff for override functionality (technical debt)\n            tag['processor'] = 'restock_diff'\n\n            self.__data['settings']['application']['tags'][uuid] = Tag.model(\n                datastore_path=self.datastore_path,\n                __datastore=self.__data,\n                default=tag\n            )\n            logger.info(f\"Tag: {uuid} {tag['title']}\")\n\n    def _rehydrate_watches(self):\n        \"\"\"Rehydrate watch entities from stored data (converts dicts to Watch objects).\"\"\"\n        watch_count = len(self.__data.get('watching', {}))\n        if watch_count == 0:\n            return\n\n        logger.info(f\"Rehydrating {watch_count} watches...\")\n        watching_rehydrated = {}\n        for uuid, watch_dict in self.__data.get('watching', {}).items():\n            if isinstance(watch_dict, dict):\n                watching_rehydrated[uuid] = self.rehydrate_entity(uuid, watch_dict)\n            else:\n                logger.error(f\"Watch UUID {uuid} already rehydrated\")\n\n        self.__data['watching'] = watching_rehydrated\n        logger.success(f\"Rehydrated {watch_count} watches into Watch objects\")\n\n\n    def _load_state(self, main_settings_filename=\"changedetection.json\"):\n        \"\"\"\n        Load complete datastore state from storage.\n\n        Orchestrates loading of settings, watches, and tags using polymorphic methods.\n        \"\"\"\n        # Load settings\n        settings_data = self._load_settings(filename=main_settings_filename)\n        self._apply_settings(settings_data)\n\n        # Load watches, scan them from the disk\n        self._load_watches()\n        self._rehydrate_watches()\n\n        # Load tags from individual tag.json files\n        # These will override any tags in settings (migration path)\n        self._load_tags()\n\n        # Rehydrate any remaining tags from settings (legacy/fallback)\n        self._rehydrate_tags()\n\n    def reload_state(self, datastore_path, include_default_watches, version_tag):\n        \"\"\"\n        Load datastore from storage or create new one.\n\n        Supports two scenarios:\n        1. NEW format: changedetection.json exists → load and run updates if needed\n        2. EMPTY: No changedetection.json → create new OR trigger migration from legacy\n\n        Note: Legacy url-watches.json migration happens in update_26, not here.\n        \"\"\"\n        logger.info(f\"Datastore path is '{datastore_path}'\")\n\n        # CRITICAL: Update datastore_path (was using old path from __init__)\n        self.datastore_path = datastore_path\n\n        # Initialize data structure\n        self.__data = App.model(datastore_path=datastore_path)\n        self.json_store_path = os.path.join(self.datastore_path, \"changedetection.json\")\n\n        # Base definition for all watchers (deepcopy part of #569)\n        self.generic_definition = deepcopy(Watch.model(datastore_path=datastore_path, __datastore=self.__data, default={}))\n\n        # Load build SHA if available (Docker deployments)\n        if path.isfile('changedetectionio/source.txt'):\n            with open('changedetectionio/source.txt') as f:\n                self.__data['build_sha'] = f.read()\n\n        # Check if datastore already exists\n        changedetection_json = os.path.join(self.datastore_path, \"changedetection.json\")\n        changedetection_json_old_schema = os.path.join(self.datastore_path, \"url-watches.json\")\n\n        if os.path.exists(changedetection_json):\n            # Run schema updates if needed\n            # Pass current schema version from loaded datastore (defaults to 0 if not set)\n            # Load existing datastore (changedetection.json + watch.json files)\n            logger.info(\"Loading existing datastore\")\n            self._load_state()\n            current_schema = self.data['settings']['application'].get('schema_version', 0)\n            self.run_updates(current_schema_version=current_schema)\n\n        # Legacy datastore detected - trigger migration, even works if the schema is much before the migration step.\n        elif os.path.exists(changedetection_json_old_schema):\n\n            logger.critical(f\"Legacy datastore detected at {changedetection_json_old_schema}, loading and running updates\")\n            self._load_state(main_settings_filename=\"url-watches.json\")\n            # update 26 will load the whole old config from disk to __data\n            current_schema = self.__data['settings']['application'].get('schema_version', 0)\n            self.run_updates(current_schema_version=current_schema)\n            # Probably tags were also shifted to disk and many other changes, so best to reload here.\n            self._load_state()\n\n        else:\n            # No datastore yet - check if this is a fresh install or legacy migration\n            self.init_fresh_install(include_default_watches=include_default_watches,\n                                    version_tag=version_tag)\n            # Maybe they copied a bunch of watch subdirs across too\n            self._load_state()\n\n    def init_fresh_install(self, include_default_watches, version_tag):\n      # Generate app_guid FIRST (required for all operations)\n        if \"pytest\" in sys.modules or \"PYTEST_CURRENT_TEST\" in os.environ:\n            self.__data['app_guid'] = \"test-\" + str(uuid_builder.uuid4())\n        else:\n            self.__data['app_guid'] = str(uuid_builder.uuid4())\n\n        # Generate RSS access token\n        self.__data['settings']['application']['rss_access_token'] = secrets.token_hex(16)\n\n        # Generate API access token\n        self.__data['settings']['application']['api_access_token'] = secrets.token_hex(16)\n        logger.warning(f\"No datastore found, creating new datastore at {self.datastore_path}\")\n\n        # Set schema version to latest (no updates needed)\n        latest_update_available = self.get_updates_available().pop()\n        logger.info(f\"Marking fresh install to schema version {latest_update_available}\")\n        self.__data['settings']['application']['schema_version'] = latest_update_available\n\n        # Add default watches if requested\n        if include_default_watches:\n            self.add_watch(\n                url='https://news.ycombinator.com/',\n                tag='Tech news',\n                extras={'fetch_backend': 'html_requests'}\n            )\n            self.add_watch(\n                url='https://changedetection.io/CHANGELOG.txt',\n                tag='changedetection.io',\n                extras={'fetch_backend': 'html_requests'}\n            )\n\n        # Create changedetection.json immediately\n        try:\n            self._save_settings()\n            logger.info(\"Created changedetection.json for new datastore\")\n        except Exception as e:\n            logger.error(f\"Failed to create initial changedetection.json: {e}\")\n\n\n\n        # Set version tag\n        self.__data['version_tag'] = version_tag\n\n        # Validate proxies.json if it exists\n        _ = self.proxy_list  # Just to test parsing\n\n        # Ensure app_guid exists (for datastores loaded from existing files)\n        if 'app_guid' not in self.__data:\n            if \"pytest\" in sys.modules or \"PYTEST_CURRENT_TEST\" in os.environ:\n                self.__data['app_guid'] = \"test-\" + str(uuid_builder.uuid4())\n            else:\n                self.__data['app_guid'] = str(uuid_builder.uuid4())\n            self.commit()\n\n        # Ensure RSS access token exists\n        if not self.__data['settings']['application'].get('rss_access_token'):\n            secret = secrets.token_hex(16)\n            self.__data['settings']['application']['rss_access_token'] = secret\n            self.commit()\n\n        # Ensure API access token exists\n        if not self.__data['settings']['application'].get('api_access_token'):\n            secret = secrets.token_hex(16)\n            self.__data['settings']['application']['api_access_token'] = secret\n            self.commit()\n\n        # Handle password reset lockfile\n        password_reset_lockfile = os.path.join(self.datastore_path, \"removepassword.lock\")\n        if path.isfile(password_reset_lockfile):\n            self.remove_password()\n            unlink(password_reset_lockfile)\n\n    def rehydrate_entity(self, uuid, entity, processor_override=None):\n        \"\"\"Set the dict back to the dict Watch object\"\"\"\n        entity['uuid'] = uuid\n\n        if processor_override:\n            watch_class = get_custom_watch_obj_for_processor(processor_override)\n            entity['processor'] = processor_override\n        else:\n            watch_class = get_custom_watch_obj_for_processor(entity.get('processor'))\n\n        if entity.get('processor') != 'text_json_diff':\n            logger.trace(f\"Loading Watch object '{watch_class.__module__}.{watch_class.__name__}' for UUID {uuid}\")\n\n        entity = watch_class(datastore_path=self.datastore_path, __datastore=self.__data, default=entity)\n        return entity\n\n    # ============================================================================\n    # FileSavingDataStore Abstract Method Implementations\n    # ============================================================================\n\n    def _watch_exists(self, uuid):\n        \"\"\"Check if watch exists in datastore.\"\"\"\n        return uuid in self.__data['watching']\n\n    def _get_watch_dict(self, uuid):\n        \"\"\"Get watch as dictionary.\"\"\"\n        return dict(self.__data['watching'][uuid])\n\n    def _build_settings_data(self):\n        \"\"\"\n        Build settings data structure for saving.\n\n        Tags behavior depends on schema version:\n        - Before update_28 (schema < 28): Tags saved in settings for migration\n        - After update_28 (schema >= 28): Tags excluded from settings (in individual files)\n\n        Returns:\n            dict: Settings data ready for serialization\n        \"\"\"\n        import copy\n\n        # Deep copy settings to avoid modifying the original\n        settings_copy = copy.deepcopy(self.__data['settings'])\n\n        # Is saved as {uuid}/tag.json\n        settings_copy['application']['tags'] = {}\n\n        return {\n            'note': 'Settings file - watches are in {uuid}/watch.json, tags are in {uuid}/tag.json',\n            'app_guid': self.__data.get('app_guid'),\n            'settings': settings_copy,\n            'build_sha': self.__data.get('build_sha'),\n            'version_tag': self.__data.get('version_tag')\n        }\n\n    def _save_settings(self):\n        \"\"\"\n        Save settings to storage.\n\n        File backend implementation: saves to changedetection.json\n        Implementation of abstract method from FileSavingDataStore.\n        Uses the generic save_json_atomic helper.\n\n        Raises:\n            OSError: If disk is full or other I/O error\n        \"\"\"\n        settings_data = self._build_settings_data()\n        changedetection_json = os.path.join(self.datastore_path, \"changedetection.json\")\n        save_json_atomic(changedetection_json, settings_data, label=\"settings\")\n\n    def _load_watches(self):\n        \"\"\"\n        Load all watches from storage.\n\n        File backend implementation: reads individual watch.json files\n        Implementation of abstract method from FileSavingDataStore.\n        Delegates to helper function and stores results in internal data structure.\n        \"\"\"\n\n        # Store loaded data\n        # @note this will also work for the old legacy format because self.__data['watching'] should already have them loaded by this point.\n        self.__data['watching'].update(load_all_watches(\n            self.datastore_path,\n            self.rehydrate_entity\n        ))\n        logger.debug(f\"Loaded {len(self.__data['watching'])} watches\")\n\n    def _load_tags(self):\n        \"\"\"\n        Load all tags from storage.\n\n        File backend implementation: reads individual tag.json files.\n        Tags loaded from files override any tags in settings (migration path).\n        \"\"\"\n        from ..model import Tag\n\n        def rehydrate_tag(uuid, entity_dict):\n            \"\"\"Rehydrate tag as Tag object with forced restock_diff processor.\"\"\"\n            entity_dict['uuid'] = uuid\n            entity_dict['processor'] = 'restock_diff'  # Force processor for override functionality\n\n            return Tag.model(\n                datastore_path=self.datastore_path,\n                __datastore=self.__data,\n                default=entity_dict\n            )\n\n        tags = load_all_tags(\n            self.datastore_path,\n            rehydrate_tag\n        )\n\n        # Override settings tags with loaded tags\n        # This ensures tag.json files take precedence over settings\n        if tags:\n            self.__data['settings']['application']['tags'].update(tags)\n            logger.info(f\"Loaded {len(tags)} tags from individual tag.json files\")\n\n    def _delete_watch(self, uuid):\n        \"\"\"\n        Delete a watch from storage.\n\n        File backend implementation: deletes entire {uuid}/ directory recursively.\n        Implementation of abstract method from FileSavingDataStore.\n\n        Args:\n            uuid: Watch UUID to delete\n        \"\"\"\n        watch_dir = os.path.join(self.datastore_path, uuid)\n        if os.path.exists(watch_dir):\n            shutil.rmtree(watch_dir)\n            logger.info(f\"Deleted watch directory: {watch_dir}\")\n\n    # ============================================================================\n    # Watch Management Methods\n    # ============================================================================\n\n    def set_last_viewed(self, uuid, timestamp):\n        logger.debug(f\"Setting watch UUID: {uuid} last viewed to {int(timestamp)}\")\n        self.data['watching'][uuid].update({'last_viewed': int(timestamp)})\n        self.data['watching'][uuid].commit()\n\n        watch_check_update = signal('watch_check_update')\n        if watch_check_update:\n            watch_check_update.send(watch_uuid=uuid)\n\n    def remove_password(self):\n        self.__data['settings']['application']['password'] = False\n        self.commit()\n\n    def clear_all_last_checksums(self):\n        \"\"\"\n        Delete all last-checksum.txt files to force reprocessing of all watches.\n\n        This should be called when global settings change, since watches inherit\n        configuration and need to reprocess even if their individual watch dict\n        hasn't been modified.\n\n        Note: We delete the checksum file rather than setting was_edited=True because:\n        - was_edited is not persisted across restarts\n        - File deletion ensures reprocessing works across app restarts\n        \"\"\"\n        deleted_count = 0\n        for uuid in self.__data['watching'].keys():\n            watch = self.__data['watching'][uuid]\n            if watch.data_dir:\n                checksum_file = os.path.join(watch.data_dir, 'last-checksum.txt')\n                if os.path.isfile(checksum_file):\n                    try:\n                        os.remove(checksum_file)\n                        deleted_count += 1\n                        logger.debug(f\"Cleared checksum for watch {uuid}\")\n                    except OSError as e:\n                        logger.warning(f\"Failed to delete checksum file for {uuid}: {e}\")\n\n        logger.info(f\"Cleared {deleted_count} checksum files to force reprocessing\")\n        return deleted_count\n\n    def clear_checksums_for_tag(self, tag_uuid):\n        \"\"\"\n        Delete last-checksum.txt files for all watches using a specific tag.\n\n        This should be called when a tag configuration is edited, since watches\n        inherit tag settings and need to reprocess.\n\n        Args:\n            tag_uuid: UUID of the tag that was modified\n\n        Returns:\n            int: Number of checksum files deleted\n        \"\"\"\n        deleted_count = 0\n        for uuid, watch in self.__data['watching'].items():\n            if watch.get('tags') and tag_uuid in watch['tags']:\n                if watch.data_dir:\n                    checksum_file = os.path.join(watch.data_dir, 'last-checksum.txt')\n                    if os.path.isfile(checksum_file):\n                        try:\n                            os.remove(checksum_file)\n                            deleted_count += 1\n                            logger.debug(f\"Cleared checksum for watch {uuid} (tag {tag_uuid})\")\n                        except OSError as e:\n                            logger.warning(f\"Failed to delete checksum file for {uuid}: {e}\")\n\n        logger.info(f\"Cleared {deleted_count} checksum files for tag {tag_uuid}\")\n        return deleted_count\n\n    def commit(self):\n        \"\"\"\n        Save settings immediately to disk using atomic write.\n\n        Uses atomic write pattern (temp file + rename) for crash safety.\n\n        Fire-and-forget: Logs errors but does not raise exceptions.\n        Settings data remains in memory even if save fails, so next commit will retry.\n        \"\"\"\n        try:\n            self._save_settings()\n            logger.debug(\"Committed settings\")\n        except Exception as e:\n            logger.error(f\"Failed to commit settings: {e}\")\n\n    def update_watch(self, uuid, update_obj):\n\n        # It's possible that the watch could be deleted before update\n        if not self.__data['watching'].get(uuid):\n            return\n\n        with self.lock:\n\n            # In python 3.9 we have the |= dict operator, but that still will lose data on nested structures...\n            for dict_key, d in self.generic_definition.items():\n                if isinstance(d, dict):\n                    if update_obj is not None and dict_key in update_obj:\n                        self.__data['watching'][uuid][dict_key].update(update_obj[dict_key])\n                        del (update_obj[dict_key])\n\n            self.__data['watching'][uuid].update(update_obj)\n\n        # Immediate save\n        self.__data['watching'][uuid].commit()\n\n    @property\n    def threshold_seconds(self):\n        seconds = 0\n        for m, n in Watch.mtable.items():\n            x = self.__data['settings']['requests']['time_between_check'].get(m)\n            if x:\n                seconds += x * n\n        return seconds\n\n    @property\n    def unread_changes_count(self):\n        unread_changes_count = 0\n        for uuid, watch in self.__data['watching'].items():\n            if watch.history_n >= 2 and watch.viewed == False:\n                unread_changes_count += 1\n\n        return unread_changes_count\n\n    @property\n    def data(self):\n        # Re #152, Return env base_url if not overriden\n        # Re #148 - Some people have just {{ base_url }} in the body or title, but this may break some notification services\n        #           like 'Join', so it's always best to atleast set something obvious so that they are not broken.\n\n        active_base_url = BASE_URL_NOT_SET_TEXT\n        if self.__data['settings']['application'].get('base_url'):\n            active_base_url = self.__data['settings']['application'].get('base_url')\n        elif os.getenv('BASE_URL'):\n            active_base_url = os.getenv('BASE_URL')\n\n        # I looked at various ways todo the following, but in the end just copying the dict seemed simplest/most reliable\n        # even given the memory tradeoff - if you know a better way.. maybe return d|self.__data.. or something\n        d = self.__data\n        d['settings']['application']['active_base_url'] = active_base_url.strip('\" ')\n        return d\n\n    # Delete a single watch by UUID\n    def delete(self, uuid):\n        \"\"\"\n        Delete a watch by UUID.\n\n        Uses abstracted storage method for backend-agnostic deletion.\n        Supports 'all' to delete all watches (mainly for testing).\n\n        Args:\n            uuid: Watch UUID to delete, or 'all' to delete all watches\n        \"\"\"\n        with self.lock:\n            if uuid == 'all':\n                # Delete all watches - capture UUIDs first before modifying dict\n                all_uuids = list(self.__data['watching'].keys())\n\n                for watch_uuid in all_uuids:\n                    # Delete from storage using polymorphic method\n                    try:\n                        self._delete_watch(watch_uuid)\n                    except Exception as e:\n                        logger.error(f\"Failed to delete watch {watch_uuid} from storage: {e}\")\n\n                    # Send delete signal\n                    watch_delete_signal = signal('watch_deleted')\n                    if watch_delete_signal:\n                        watch_delete_signal.send(watch_uuid=watch_uuid)\n\n                # Clear the dict\n                self.__data['watching'] = {}\n\n                # Mainly used for testing to allow all items to flush before running next test\n                time.sleep(1)\n\n            else:\n                # Delete single watch from storage using polymorphic method\n                try:\n                    self._delete_watch(uuid)\n                except Exception as e:\n                    logger.error(f\"Failed to delete watch {uuid} from storage: {e}\")\n\n                # Remove from watching dict\n                del self.data['watching'][uuid]\n\n                # Send delete signal\n                watch_delete_signal = signal('watch_deleted')\n                if watch_delete_signal:\n                    watch_delete_signal.send(watch_uuid=uuid)\n\n    # Clone a watch by UUID\n    def clone(self, uuid):\n        url = self.data['watching'][uuid].get('url')\n        # No need to deepcopy here - add_watch() will deepcopy extras anyway (line 569)\n        # Just pass a dict copy (with lock for thread safety)\n        # NOTE: dict() is shallow copy but safe since add_watch() deepcopies it\n        with self.lock:\n            extras = dict(self.data['watching'][uuid])\n        new_uuid = self.add_watch(url=url, extras=extras)\n        watch = self.data['watching'][new_uuid]\n        return new_uuid\n\n    def url_exists(self, url):\n\n        # Probably their should be dict...\n        for watch in self.data['watching'].values():\n            if watch['url'].lower() == url.lower():\n                return True\n\n        return False\n\n    # Remove a watchs data but keep the entry (URL etc)\n    def clear_watch_history(self, uuid):\n        self.__data['watching'][uuid].clear_watch()\n        self.__data['watching'][uuid].commit()\n\n    def add_watch(self, url, tag='', extras=None, tag_uuids=None, save_immediately=True):\n\n        if extras is None:\n            extras = {}\n\n        # Incase these are copied across, assume it's a reference and deepcopy()\n        apply_extras = deepcopy(extras)\n        apply_extras['tags'] = [] if not apply_extras.get('tags') else apply_extras.get('tags')\n\n        # Was it a share link? try to fetch the data\n        if (url.startswith(\"https://changedetection.io/share/\")):\n            import requests\n\n            try:\n                r = requests.request(method=\"GET\",\n                                     url=url,\n                                     # So we know to return the JSON instead of the human-friendly \"help\" page\n                                     headers={'App-Guid': self.__data['app_guid']},\n                                     timeout=5.0)  # 5 second timeout to prevent blocking\n                res = r.json()\n\n                # List of permissible attributes we accept from the wild internet\n                for k in [\n                    'body',\n                    'browser_steps',\n                    'css_filter',\n                    'extract_text',\n                    'headers',\n                    'ignore_text',\n                    'include_filters',\n                    'method',\n                    'paused',\n                    'previous_md5',\n                    'processor',\n                    'subtractive_selectors',\n                    'tag',\n                    'tags',\n                    'text_should_not_be_present',\n                    'title',\n                    'trigger_text',\n                    'url',\n                    'use_page_title_in_list',\n                    'webdriver_js_execute_code',\n                ]:\n                    if res.get(k):\n                        if k != 'css_filter':\n                            apply_extras[k] = res[k]\n                        else:\n                            # We renamed the field and made it a list\n                            apply_extras['include_filters'] = [res['css_filter']]\n\n            except Exception as e:\n                logger.error(f\"Error fetching metadata for shared watch link {url} {str(e)}\")\n                flash(gettext(\"Error fetching metadata for {}\").format(url), 'error')\n                return False\n\n        if not is_safe_valid_url(url):\n            from flask import has_request_context\n            if has_request_context():\n                flash(gettext('Watch protocol is not permitted or invalid URL format'), 'error')\n            else:\n                logger.error(f\"add_watch: URL '{url}' is not permitted or invalid, skipping.\")\n            return None\n\n        # Check PAGE_WATCH_LIMIT if set\n        page_watch_limit = os.getenv('PAGE_WATCH_LIMIT')\n        if page_watch_limit:\n            try:\n                page_watch_limit = int(page_watch_limit)\n                current_watch_count = len(self.__data['watching'])\n                if current_watch_count >= page_watch_limit:\n                    logger.error(f\"Watch limit reached: {current_watch_count}/{page_watch_limit} watches. Cannot add {url}\")\n                    flash(gettext(\"Watch limit reached ({}/{} watches). Cannot add more watches.\").format(current_watch_count, page_watch_limit), 'error')\n                    return None\n            except ValueError:\n                logger.warning(f\"Invalid PAGE_WATCH_LIMIT value: {page_watch_limit}, ignoring limit check\")\n\n        if tag and type(tag) == str:\n            # Then it's probably a string of the actual tag by name, split and add it\n            for t in tag.split(','):\n                # for each stripped tag, add tag as UUID\n                for a_t in t.split(','):\n                    tag_uuid = self.add_tag(a_t)\n                    apply_extras['tags'].append(tag_uuid)\n\n        # Or if UUIDs given directly\n        if tag_uuids:\n            for t in tag_uuids:\n                apply_extras['tags'] = list(set(apply_extras['tags'] + [t.strip()]))\n\n        # Make any uuids unique\n        if apply_extras.get('tags'):\n            apply_extras['tags'] = list(set(apply_extras.get('tags')))\n\n        # If the processor also has its own Watch implementation\n        watch_class = get_custom_watch_obj_for_processor(apply_extras.get('processor'))\n        new_watch = watch_class(datastore_path=self.datastore_path, __datastore=self.__data, url=url)\n\n        new_uuid = new_watch.get('uuid')\n\n        logger.debug(f\"Adding URL '{url}' - {new_uuid}\")\n\n        for k in ['uuid', 'history', 'last_checked', 'last_changed', 'newest_history_key', 'previous_md5', 'viewed']:\n            if k in apply_extras:\n                del apply_extras[k]\n\n        if not apply_extras.get('date_created'):\n            apply_extras['date_created'] = int(time.time())\n\n        new_watch.update(apply_extras)\n        new_watch.ensure_data_dir_exists()\n        self.__data['watching'][new_uuid] = new_watch\n\n        if save_immediately:\n            # Save immediately using commit\n            new_watch.commit()\n            logger.debug(f\"Saved new watch {new_uuid}\")\n\n        logger.debug(f\"Added '{url}'\")\n\n        return new_uuid\n\n    def _watch_resource_exists(self, watch_uuid, resource_name):\n        \"\"\"\n        Check if a watch-related resource exists.\n\n        File backend implementation: checks if file exists in watch directory.\n\n        Args:\n            watch_uuid: Watch UUID\n            resource_name: Name of resource (e.g., \"last-screenshot.png\")\n\n        Returns:\n            bool: True if resource exists\n        \"\"\"\n        resource_path = os.path.join(self.datastore_path, watch_uuid, resource_name)\n        return path.isfile(resource_path)\n\n    def visualselector_data_is_ready(self, watch_uuid):\n        \"\"\"\n        Check if visual selector data (screenshot + elements) is ready.\n\n        Returns:\n            bool: True if both screenshot and elements data exist\n        \"\"\"\n        has_screenshot = self._watch_resource_exists(watch_uuid, \"last-screenshot.png\")\n        has_elements = self._watch_resource_exists(watch_uuid, \"elements.deflate\")\n        return has_screenshot and has_elements\n\n    # Old sync_to_json and save_datastore methods removed - now handled by FileSavingDataStore parent class\n\n    @property\n    def proxy_list(self):\n        proxy_list = {}\n        proxy_list_file = os.path.join(self.datastore_path, 'proxies.json')\n\n        # Load from external config file\n        if path.isfile(proxy_list_file):\n            if HAS_ORJSON:\n                # orjson.loads() expects UTF-8 encoded bytes #3611\n                with open(os.path.join(self.datastore_path, \"proxies.json\"), 'rb') as f:\n                    proxy_list = orjson.loads(f.read())\n            else:\n                with open(os.path.join(self.datastore_path, \"proxies.json\"), encoding='utf-8') as f:\n                    proxy_list = json.load(f)\n\n        # Mapping from UI config if available\n        extras = self.data['settings']['requests'].get('extra_proxies')\n        if extras:\n            i = 0\n            for proxy in extras:\n                i += 0\n                if proxy.get('proxy_name') and proxy.get('proxy_url'):\n                    k = \"ui-\" + str(i) + proxy.get('proxy_name')\n                    proxy_list[k] = {'label': proxy.get('proxy_name'), 'url': proxy.get('proxy_url')}\n\n        if proxy_list and strtobool(os.getenv('ENABLE_NO_PROXY_OPTION', 'True')):\n            proxy_list[\"no-proxy\"] = {'label': \"No proxy\", 'url': ''}\n\n        return proxy_list if len(proxy_list) else None\n\n    def get_preferred_proxy_for_watch(self, uuid):\n        \"\"\"\n        Returns the preferred proxy by ID key\n        :param uuid: UUID\n        :return: proxy \"key\" id\n        \"\"\"\n\n        if self.proxy_list is None:\n            return None\n\n        # If it's a valid one\n        watch = self.data['watching'].get(uuid)\n\n        if strtobool(os.getenv('ENABLE_NO_PROXY_OPTION', 'True')) and watch.get('proxy') == \"no-proxy\":\n            return None\n\n        if watch.get('proxy') and watch.get('proxy') in list(self.proxy_list.keys()):\n            return watch.get('proxy')\n\n        # not valid (including None), try the system one\n        else:\n            system_proxy_id = self.data['settings']['requests'].get('proxy')\n            # Is not None and exists\n            if self.proxy_list.get(system_proxy_id):\n                return system_proxy_id\n\n        # Fallback - Did not resolve anything, or doesnt exist, use the first available\n        if system_proxy_id is None or not self.proxy_list.get(system_proxy_id):\n            first_default = list(self.proxy_list)[0]\n            return first_default\n\n        return None\n\n    @property\n    def has_extra_headers_file(self):\n        filepath = os.path.join(self.datastore_path, 'headers.txt')\n        return os.path.isfile(filepath)\n\n    def get_all_base_headers(self):\n        headers = {}\n        # Global app settings\n        headers.update(self.data['settings'].get('headers', {}))\n\n        return headers\n\n    def get_all_headers_in_textfile_for_watch(self, uuid):\n        from ..model.App import parse_headers_from_text_file\n        headers = {}\n\n        # Global in /datastore/headers.txt\n        filepath = os.path.join(self.datastore_path, 'headers.txt')\n        try:\n            if os.path.isfile(filepath):\n                headers.update(parse_headers_from_text_file(filepath))\n        except Exception as e:\n            logger.error(f\"ERROR reading headers.txt at {filepath} {str(e)}\")\n\n        watch = self.data['watching'].get(uuid)\n        if watch:\n\n            # In /datastore/xyz-xyz/headers.txt\n            filepath = os.path.join(watch.data_dir, 'headers.txt')\n            try:\n                if os.path.isfile(filepath):\n                    headers.update(parse_headers_from_text_file(filepath))\n            except Exception as e:\n                logger.error(f\"ERROR reading headers.txt at {filepath} {str(e)}\")\n\n            # In /datastore/tag-name.txt\n            tags = self.get_all_tags_for_watch(uuid=uuid)\n            for tag_uuid, tag in tags.items():\n                fname = \"headers-\" + re.sub(r'[\\W_]', '', tag.get('title')).lower().strip() + \".txt\"\n                filepath = os.path.join(self.datastore_path, fname)\n                try:\n                    if os.path.isfile(filepath):\n                        headers.update(parse_headers_from_text_file(filepath))\n                except Exception as e:\n                    logger.error(f\"ERROR reading headers.txt at {filepath} {str(e)}\")\n\n        return headers\n\n    def get_tag_overrides_for_watch(self, uuid, attr):\n        tags = self.get_all_tags_for_watch(uuid=uuid)\n        ret = []\n\n        if tags:\n            for tag_uuid, tag in tags.items():\n                if attr in tag and tag[attr]:\n                    ret = [*ret, *tag[attr]]\n\n        return ret\n\n    def add_tag(self, title):\n        # If name exists, return that\n        n = title.strip().lower()\n        logger.debug(f\">>> Adding new tag - '{n}'\")\n        if not n:\n            return False\n\n        for uuid, tag in self.__data['settings']['application'].get('tags', {}).items():\n            if n == tag.get('title', '').lower().strip():\n                logger.warning(f\"Tag '{title}' already exists, skipping creation.\")\n                return uuid\n\n        # Eventually almost everything todo with a watch will apply as a Tag\n        # So we use the same model as a Watch\n        with self.lock:\n            from ..model import Tag\n            new_tag = Tag.model(\n                datastore_path=self.datastore_path,\n                __datastore=self.__data,\n                default={\n                    'title': title.strip(),\n                    'date_created': int(time.time())\n                }\n            )\n\n            new_uuid = new_tag.get('uuid')\n\n            self.__data['settings']['application']['tags'][new_uuid] = new_tag\n\n        # Save tag to its own tag.json file instead of settings\n        new_tag.commit()\n        return new_uuid\n\n    def get_all_tags_for_watch(self, uuid):\n        \"\"\"This should be in Watch model but Watch doesn't have access to datastore, not sure how to solve that yet\"\"\"\n        watch = self.data['watching'].get(uuid)\n\n        # Should return a dict of full tag info linked by UUID\n        if watch:\n            return dictfilt(self.__data['settings']['application']['tags'], watch.get('tags', []))\n\n        return {}\n\n    @property\n    def extra_browsers(self):\n        res = []\n        p = list(filter(\n            lambda s: (s.get('browser_name') and s.get('browser_connection_url')),\n            self.__data['settings']['requests'].get('extra_browsers', [])))\n        if p:\n            for i in p:\n                res.append((\"extra_browser_\" + i['browser_name'], i['browser_name']))\n\n        return res\n\n    def tag_exists_by_name(self, tag_name):\n        # Check if any tag dictionary has a 'title' attribute matching the provided tag_name\n        tags = self.__data['settings']['application']['tags'].values()\n        return next((v for v in tags if v.get('title', '').lower() == tag_name.lower()),\n                    None)\n\n    def any_watches_have_processor_by_name(self, processor_name):\n        for watch in self.data['watching'].values():\n            if watch.get('processor') == processor_name:\n                return True\n        return False\n\n    def search_watches_for_url(self, query, tag_limit=None, partial=False):\n        \"\"\"Search watches by URL, title, or error messages\n\n        Args:\n            query (str): Search term to match against watch URLs, titles, and error messages\n            tag_limit (str, optional): Optional tag name to limit search results\n            partial: (bool, optional): sub-string matching\n\n        Returns:\n            list: List of UUIDs of watches that match the search criteria\n        \"\"\"\n        matching_uuids = []\n        query = query.lower().strip()\n        tag = self.tag_exists_by_name(tag_limit) if tag_limit else False\n\n        for uuid, watch in self.data['watching'].items():\n            # Filter by tag if requested\n            if tag_limit:\n                if not tag.get('uuid') in watch.get('tags', []):\n                    continue\n\n            # Search in URL, title, or error messages\n            if partial:\n                if ((watch.get('title') and query in watch.get('title').lower()) or\n                        query in watch.get('url', '').lower() or\n                        (watch.get('last_error') and query in watch.get('last_error').lower())):\n                    matching_uuids.append(uuid)\n            else:\n                if ((watch.get('title') and query == watch.get('title').lower()) or\n                        query == watch.get('url', '').lower() or\n                        (watch.get('last_error') and query == watch.get('last_error').lower())):\n                    matching_uuids.append(uuid)\n\n        return matching_uuids\n\n    def get_unique_notification_tokens_available(self):\n        # Ask each type of watch if they have any extra notification token to add to the validation\n        extra_notification_tokens = {}\n        watch_processors_checked = set()\n\n        for watch_uuid, watch in self.__data['watching'].items():\n            processor = watch.get('processor')\n            if processor not in watch_processors_checked:\n                extra_notification_tokens.update(watch.extra_notification_token_values())\n                watch_processors_checked.add(processor)\n\n        return extra_notification_tokens\n\n    def get_unique_notification_token_placeholders_available(self):\n        # The actual description of the tokens, could be combined with get_unique_notification_tokens_available instead of doing this twice\n        extra_notification_tokens = []\n        watch_processors_checked = set()\n\n        for watch_uuid, watch in self.__data['watching'].items():\n            processor = watch.get('processor')\n            if processor not in watch_processors_checked:\n                extra_notification_tokens += watch.extra_notification_token_placeholder_info()\n                watch_processors_checked.add(processor)\n\n        return extra_notification_tokens\n\n    def add_notification_url(self, notification_url):\n\n        logger.debug(f\">>> Adding new notification_url - '{notification_url}'\")\n\n        notification_urls = self.data['settings']['application'].get('notification_urls', [])\n\n        if notification_url in notification_urls:\n            return notification_url\n\n        with self.lock:\n            notification_urls = self.__data['settings']['application'].get('notification_urls', [])\n\n            if notification_url in notification_urls:\n                return notification_url\n\n            # Append and update the datastore\n            notification_urls.append(notification_url)\n            self.__data['settings']['application']['notification_urls'] = notification_urls\n\n        self.commit()\n        return notification_url\n\n    # Schema update methods moved to store/updates.py (DatastoreUpdatesMixin)\n    # This includes: get_updates_available(), run_updates(), and update_1() through update_26()\n"
  },
  {
    "path": "changedetectionio/store/base.py",
    "content": "\"\"\"\nBase classes for the datastore.\n\nThis module defines the abstract interfaces that all datastore implementations must follow.\n\"\"\"\n\nfrom abc import ABC, abstractmethod\nfrom threading import Lock\nfrom loguru import logger\n\n\nclass DataStore(ABC):\n    \"\"\"\n    Abstract base class for all datastore implementations.\n\n    Defines the core interface that all datastores must implement for:\n    - Loading and saving data\n    - Managing watches\n    - Handling settings\n    - Providing data access\n    \"\"\"\n\n    lock = Lock()\n    datastore_path = None\n\n    @abstractmethod\n    def reload_state(self, datastore_path, include_default_watches, version_tag):\n        \"\"\"\n        Load data from persistent storage.\n\n        Args:\n            datastore_path: Path to the datastore directory\n            include_default_watches: Whether to create default watches if none exist\n            version_tag: Application version string\n        \"\"\"\n        pass\n\n    @abstractmethod\n    def add_watch(self, url, **kwargs):\n        \"\"\"\n        Add a new watch.\n\n        Args:\n            url: URL to watch\n            **kwargs: Additional watch parameters\n\n        Returns:\n            UUID of the created watch\n        \"\"\"\n        pass\n\n    @abstractmethod\n    def update_watch(self, uuid, update_obj):\n        \"\"\"\n        Update an existing watch.\n\n        Args:\n            uuid: Watch UUID\n            update_obj: Dictionary of fields to update\n        \"\"\"\n        pass\n\n    @abstractmethod\n    def delete(self, uuid):\n        \"\"\"\n        Delete a watch.\n\n        Args:\n            uuid: Watch UUID to delete\n        \"\"\"\n        pass\n\n    @property\n    @abstractmethod\n    def data(self):\n        \"\"\"\n        Access to the underlying data structure.\n\n        Returns:\n            Dictionary containing all datastore data\n        \"\"\"\n        pass\n\n"
  },
  {
    "path": "changedetectionio/store/file_saving_datastore.py",
    "content": "\"\"\"\nFile-based datastore with individual watch persistence and immediate commits.\n\nThis module provides the FileSavingDataStore abstract class that implements:\n- Individual watch.json file persistence\n- Immediate commit-based persistence (watch.commit(), datastore.commit())\n- Atomic file writes safe for NFS/NAS\n\"\"\"\n\nimport glob\nimport json\nimport os\nimport tempfile\nimport time\nfrom loguru import logger\n\nfrom .base import DataStore\nfrom .. import strtobool\n\n# Try to import orjson for faster JSON serialization\ntry:\n    import orjson\n    HAS_ORJSON = True\nexcept ImportError:\n    HAS_ORJSON = False\n\n# Fsync configuration: Force file data to disk for crash safety\n# Default False to match legacy behavior (write-and-rename without fsync)\n# Set to True for mission-critical deployments requiring crash consistency\nFORCE_FSYNC_DATA_IS_CRITICAL = bool(strtobool(os.getenv('FORCE_FSYNC_DATA_IS_CRITICAL', 'False')))\n\n# ============================================================================\n# Helper Functions for Atomic File Operations\n# ============================================================================\n\ndef save_json_atomic(file_path, data_dict, label=\"file\", max_size_mb=10):\n    \"\"\"\n    Save JSON data to disk using atomic write pattern.\n\n    Generic helper for saving any JSON data (settings, watches, etc.) with:\n    - Atomic write (temp file + rename)\n    - Directory fsync for crash consistency (only for new files)\n    - Size validation\n    - Proper error handling\n\n    Thread safety: Caller must hold datastore.lock to prevent concurrent modifications.\n    Multi-process safety: Not supported - run only one app instance per datastore.\n\n    Args:\n        file_path: Full path to target JSON file\n        data_dict: Dictionary to serialize\n        label: Human-readable label for error messages (e.g., \"watch\", \"settings\")\n        max_size_mb: Maximum allowed file size in MB\n\n    Raises:\n        ValueError: If serialized data exceeds max_size_mb\n        OSError: If disk is full (ENOSPC) or other I/O error\n    \"\"\"\n    # Check if file already exists (before we start writing)\n    # Directory fsync only needed for NEW files to persist the filename\n    file_exists = os.path.exists(file_path)\n\n    # Ensure parent directory exists\n    parent_dir = os.path.dirname(file_path)\n    os.makedirs(parent_dir, exist_ok=True)\n\n    # Create temp file in same directory (required for NFS atomicity)\n    fd, temp_path = tempfile.mkstemp(\n        suffix='.tmp',\n        prefix='json-',\n        dir=parent_dir,\n        text=False\n    )\n\n    fd_closed = False\n    try:\n        # Serialize data\n        t0 = time.time()\n        if HAS_ORJSON:\n            data = orjson.dumps(data_dict, option=orjson.OPT_INDENT_2)\n        else:\n            data = json.dumps(data_dict, indent=2, ensure_ascii=False).encode('utf-8')\n        serialize_ms = (time.time() - t0) * 1000\n\n        # Safety check: validate size\n        MAX_SIZE = max_size_mb * 1024 * 1024\n        data_size = len(data)\n        if data_size > MAX_SIZE:\n            raise ValueError(\n                f\"{label.capitalize()} data is unexpectedly large: {data_size / 1024 / 1024:.2f}MB \"\n                f\"(max: {max_size_mb}MB). This indicates a bug or data corruption.\"\n            )\n\n        # Write to temp file\n        t1 = time.time()\n        os.write(fd, data)\n        write_ms = (time.time() - t1) * 1000\n\n        # Optional fsync: Force file data to disk for crash safety\n        # Only if FORCE_FSYNC_DATA_IS_CRITICAL=True (default: False, matches legacy behavior)\n        t2 = time.time()\n        if FORCE_FSYNC_DATA_IS_CRITICAL:\n            os.fsync(fd)\n        file_fsync_ms = (time.time() - t2) * 1000\n\n        os.close(fd)\n        fd_closed = True\n\n        # Atomic rename\n        t3 = time.time()\n        os.replace(temp_path, file_path)\n        rename_ms = (time.time() - t3) * 1000\n\n        # Sync directory to ensure filename metadata is durable\n        # OPTIMIZATION: Only needed for NEW files. Existing files already have\n        # directory entry persisted, so we only need file fsync for data durability.\n        dir_fsync_ms = 0\n        if not file_exists:\n            try:\n                dir_fd = os.open(parent_dir, os.O_RDONLY)\n                try:\n                    t4 = time.time()\n                    os.fsync(dir_fd)\n                    dir_fsync_ms = (time.time() - t4) * 1000\n                finally:\n                    os.close(dir_fd)\n            except (OSError, AttributeError):\n                # Windows doesn't support fsync on directories\n                pass\n\n        # Log timing breakdown for slow saves\n#        total_ms = serialize_ms + write_ms + file_fsync_ms + rename_ms + dir_fsync_ms\n#        if total_ms:  # Log if save took more than 10ms\n#            file_status = \"new\" if not file_exists else \"update\"\n#            logger.trace(\n#                f\"Save timing breakdown ({total_ms:.1f}ms total, {file_status}): \"\n#                f\"serialize={serialize_ms:.1f}ms, write={write_ms:.1f}ms, \"\n#                f\"file_fsync={file_fsync_ms:.1f}ms, rename={rename_ms:.1f}ms, \"\n#                f\"dir_fsync={dir_fsync_ms:.1f}ms, using_orjson={HAS_ORJSON}\"\n#            )\n\n    except OSError as e:\n        # Cleanup temp file\n        if not fd_closed:\n            try:\n                os.close(fd)\n            except:\n                pass\n        if os.path.exists(temp_path):\n            try:\n                os.unlink(temp_path)\n            except:\n                pass\n\n        # Provide helpful error messages\n        if e.errno == 28:  # ENOSPC\n            raise OSError(f\"Disk full: Cannot save {label}\") from e\n        elif e.errno == 122:  # EDQUOT\n            raise OSError(f\"Disk quota exceeded: Cannot save {label}\") from e\n        else:\n            raise OSError(f\"I/O error saving {label}: {e}\") from e\n\n    except Exception as e:\n        # Cleanup temp file\n        if not fd_closed:\n            try:\n                os.close(fd)\n            except:\n                pass\n        if os.path.exists(temp_path):\n            try:\n                os.unlink(temp_path)\n            except:\n                pass\n        raise e\n\n\ndef save_entity_atomic(entity_dir, uuid, entity_dict, filename, entity_type, max_size_mb):\n    \"\"\"\n    Save an entity (watch/tag) to disk using atomic write pattern.\n\n    Generic function for saving any watch_base subclass (Watch, Tag, etc.).\n\n    Args:\n        entity_dir: Directory for this entity (e.g., /datastore/{uuid})\n        uuid: Entity UUID (for logging)\n        entity_dict: Dictionary representation of the entity\n        filename: JSON filename (e.g., 'watch.json', 'tag.json')\n        entity_type: Type label for logging (e.g., 'watch', 'tag')\n        max_size_mb: Maximum allowed file size in MB\n\n    Raises:\n        ValueError: If serialized data exceeds max_size_mb\n        OSError: If disk is full (ENOSPC) or other I/O error\n    \"\"\"\n    entity_json = os.path.join(entity_dir, filename)\n    save_json_atomic(entity_json, entity_dict, label=f\"{entity_type} {uuid}\", max_size_mb=max_size_mb)\n\n\ndef save_watch_atomic(watch_dir, uuid, watch_dict):\n    \"\"\"\n    Save a watch to disk using atomic write pattern.\n\n    Convenience wrapper around save_entity_atomic for watches.\n    Kept for backwards compatibility.\n    \"\"\"\n    save_entity_atomic(watch_dir, uuid, watch_dict, \"watch.json\", \"watch\", max_size_mb=10)\n\n\n\ndef load_watch_from_file(watch_json, uuid, rehydrate_entity_func):\n    \"\"\"\n    Load a watch from its JSON file.\n\n    Args:\n        watch_json: Path to the watch.json file\n        uuid: Watch UUID\n        rehydrate_entity_func: Function to convert dict to Watch object\n\n    Returns:\n        Watch object or None if failed\n    \"\"\"\n    try:\n        # Check file size before reading\n        file_size = os.path.getsize(watch_json)\n        MAX_WATCH_SIZE = 10 * 1024 * 1024  # 10MB\n        if file_size > MAX_WATCH_SIZE:\n            logger.critical(\n                f\"CORRUPTED WATCH DATA: Watch {uuid} file is unexpectedly large: \"\n                f\"{file_size / 1024 / 1024:.2f}MB (max: {MAX_WATCH_SIZE / 1024 / 1024}MB). \"\n                f\"File: {watch_json}. This indicates a bug or data corruption. \"\n                f\"Watch will be skipped.\"\n            )\n            return None\n\n        if HAS_ORJSON:\n            with open(watch_json, 'rb') as f:\n                watch_data = orjson.loads(f.read())\n        else:\n            with open(watch_json, 'r', encoding='utf-8') as f:\n                watch_data = json.load(f)\n\n        # Rehydrate and return watch object\n        watch_obj = rehydrate_entity_func(uuid, watch_data)\n        return watch_obj\n\n    except json.JSONDecodeError as e:\n        logger.critical(\n            f\"CORRUPTED WATCH DATA: Failed to parse JSON for watch {uuid}. \"\n            f\"File: {watch_json}. Error: {e}. \"\n            f\"Watch will be skipped and may need manual recovery from backup.\"\n        )\n        return None\n    except ValueError as e:\n        # orjson raises ValueError for invalid JSON\n        if \"invalid json\" in str(e).lower() or HAS_ORJSON:\n            logger.critical(\n                f\"CORRUPTED WATCH DATA: Failed to parse JSON for watch {uuid}. \"\n                f\"File: {watch_json}. Error: {e}. \"\n                f\"Watch will be skipped and may need manual recovery from backup.\"\n            )\n            return None\n        # Re-raise if it's not a JSON parsing error\n        raise\n    except FileNotFoundError:\n        logger.error(f\"Watch file not found: {watch_json} for watch {uuid}\")\n        return None\n    except Exception as e:\n        logger.error(f\"Failed to load watch {uuid} from {watch_json}: {e}\")\n        return None\n\n\ndef load_all_watches(datastore_path, rehydrate_entity_func):\n    \"\"\"\n    Load all watches from individual watch.json files.\n\n    SYNCHRONOUS loading: Blocks until all watches are loaded.\n    This ensures data consistency - web server won't accept requests\n    until all watches are available. Progress logged every 100 watches.\n\n    Args:\n        datastore_path: Path to the datastore directory\n        rehydrate_entity_func: Function to convert dict to Watch object\n\n    Returns:\n        Dictionary of uuid -> Watch object\n    \"\"\"\n    start_time = time.time()\n    logger.info(\"Loading watches from individual watch.json files...\")\n\n    watching = {}\n\n    if not os.path.exists(datastore_path):\n        return watching\n\n    # Find all watch.json files using glob (faster than manual directory traversal)\n    glob_start = time.time()\n    watch_files = glob.glob(os.path.join(datastore_path, \"*\", \"watch.json\"))\n    glob_time = time.time() - glob_start\n\n    total = len(watch_files)\n    logger.debug(f\"Found {total} watch.json files in {glob_time:.3f}s\")\n\n    loaded = 0\n    failed = 0\n\n    for watch_json in watch_files:\n        # Extract UUID from path: /datastore/{uuid}/watch.json\n        uuid_dir = os.path.basename(os.path.dirname(watch_json))\n        watch = load_watch_from_file(watch_json, uuid_dir, rehydrate_entity_func)\n        if watch:\n            watching[uuid_dir] = watch\n            loaded += 1\n\n            if loaded % 100 == 0:\n                logger.info(f\"Loaded {loaded}/{total} watches...\")\n        else:\n            # load_watch_from_file already logged the specific error\n            failed += 1\n\n    elapsed = time.time() - start_time\n\n    if failed > 0:\n        logger.critical(\n            f\"LOAD COMPLETE: {loaded} watches loaded successfully, \"\n            f\"{failed} watches FAILED to load (corrupted or invalid) \"\n            f\"in {elapsed:.2f}s ({loaded/elapsed:.0f} watches/sec)\"\n        )\n    else:\n        logger.info(f\"Loaded {loaded} watches from disk in {elapsed:.2f}s ({loaded/elapsed:.0f} watches/sec)\")\n\n    return watching\n\n\ndef load_tag_from_file(tag_json, uuid, rehydrate_entity_func):\n    \"\"\"\n    Load a tag from its JSON file.\n\n    Args:\n        tag_json: Path to the tag.json file\n        uuid: Tag UUID\n        rehydrate_entity_func: Function to convert dict to Tag object\n\n    Returns:\n        Tag object or None if failed\n    \"\"\"\n    try:\n        # Check file size before reading\n        file_size = os.path.getsize(tag_json)\n        MAX_TAG_SIZE = 1 * 1024 * 1024  # 1MB\n        if file_size > MAX_TAG_SIZE:\n            logger.critical(\n                f\"CORRUPTED TAG DATA: Tag {uuid} file is unexpectedly large: \"\n                f\"{file_size / 1024 / 1024:.2f}MB (max: {MAX_TAG_SIZE / 1024 / 1024}MB). \"\n                f\"File: {tag_json}. This indicates a bug or data corruption. \"\n                f\"Tag will be skipped.\"\n            )\n            return None\n\n        if HAS_ORJSON:\n            with open(tag_json, 'rb') as f:\n                tag_data = orjson.loads(f.read())\n        else:\n            with open(tag_json, 'r', encoding='utf-8') as f:\n                tag_data = json.load(f)\n\n        tag_data['processor'] = 'restock_diff'\n        # Rehydrate tag (convert dict to Tag object)\n        # processor_override is set inside the rehydration function\n        tag_obj = rehydrate_entity_func(uuid, tag_data)\n        return tag_obj\n\n    except json.JSONDecodeError as e:\n        logger.critical(\n            f\"CORRUPTED TAG DATA: Failed to parse JSON for tag {uuid}. \"\n            f\"File: {tag_json}. Error: {e}. \"\n            f\"Tag will be skipped and may need manual recovery from backup.\"\n        )\n        return None\n    except ValueError as e:\n        # orjson raises ValueError for invalid JSON\n        if \"invalid json\" in str(e).lower() or HAS_ORJSON:\n            logger.critical(\n                f\"CORRUPTED TAG DATA: Failed to parse JSON for tag {uuid}. \"\n                f\"File: {tag_json}. Error: {e}. \"\n                f\"Tag will be skipped and may need manual recovery from backup.\"\n            )\n            return None\n        # Re-raise if it's not a JSON parsing error\n        raise\n    except FileNotFoundError:\n        logger.debug(f\"Tag file not found: {tag_json} for tag {uuid}\")\n        return None\n    except Exception as e:\n        logger.error(f\"Failed to load tag {uuid} from {tag_json}: {e}\")\n        return None\n\n\ndef load_all_tags(datastore_path, rehydrate_entity_func):\n    \"\"\"\n    Load all tags from individual tag.json files.\n\n    Tags are stored separately from settings in {uuid}/tag.json files.\n\n    Args:\n        datastore_path: Path to the datastore directory\n        rehydrate_entity_func: Function to convert dict to Tag object\n\n    Returns:\n        Dictionary of uuid -> Tag object\n    \"\"\"\n    logger.info(\"Loading tags from individual tag.json files...\")\n\n    tags = {}\n\n    if not os.path.exists(datastore_path):\n        return tags\n\n    # Find all tag.json files using glob\n    tag_files = glob.glob(os.path.join(datastore_path, \"*\", \"tag.json\"))\n\n    total = len(tag_files)\n    if total == 0:\n        logger.debug(\"No tag.json files found\")\n        return tags\n\n    logger.debug(f\"Found {total} tag.json files\")\n\n    loaded = 0\n    failed = 0\n\n    for tag_json in tag_files:\n        # Extract UUID from path: /datastore/{uuid}/tag.json\n        uuid_dir = os.path.basename(os.path.dirname(tag_json))\n        tag = load_tag_from_file(tag_json, uuid_dir, rehydrate_entity_func)\n        if tag:\n            tags[uuid_dir] = tag\n            loaded += 1\n        else:\n            # load_tag_from_file already logged the specific error\n            failed += 1\n\n    if failed > 0:\n        logger.warning(f\"Loaded {loaded} tags, {failed} tags FAILED to load\")\n    else:\n        logger.info(f\"Loaded {loaded} tags from disk\")\n\n    return tags\n\n\n# ============================================================================\n# FileSavingDataStore Class\n# ============================================================================\n\nclass FileSavingDataStore(DataStore):\n    \"\"\"\n    Abstract datastore that provides file persistence with immediate commits.\n\n    Features:\n    - Individual watch.json files (one per watch)\n    - Immediate persistence via watch.commit() and datastore.commit()\n    - Atomic file writes for crash safety\n\n    Subclasses must implement:\n    - rehydrate_entity(): Convert dict to Watch object\n    - Access to internal __data structure for watch management\n    \"\"\"\n\n    def __init__(self):\n        super().__init__()\n\n    def _save_settings(self):\n        \"\"\"\n        Save settings to storage (polymorphic).\n\n        Subclasses must implement for their backend.\n        - File: changedetection.json\n        - Redis: SET settings\n        - SQL: UPDATE settings table\n        \"\"\"\n        raise NotImplementedError(\"Subclass must implement _save_settings\")\n\n\n    def _load_watches(self):\n        \"\"\"\n        Load all watches from storage (polymorphic).\n\n        Subclasses must implement for their backend.\n        - File: Read individual watch.json files\n        - Redis: SCAN watch:* keys\n        - SQL: SELECT * FROM watches\n        \"\"\"\n        raise NotImplementedError(\"Subclass must implement _load_watches\")\n\n    def _delete_watch(self, uuid):\n        \"\"\"\n        Delete a watch from storage (polymorphic).\n\n        Subclasses must implement for their backend.\n        - File: Delete {uuid}/ directory recursively\n        - Redis: DEL watch:{uuid}\n        - SQL: DELETE FROM watches WHERE uuid=?\n\n        Args:\n            uuid: Watch UUID to delete\n        \"\"\"\n        raise NotImplementedError(\"Subclass must implement _delete_watch\")\n\n\n"
  },
  {
    "path": "changedetectionio/store/updates.py",
    "content": "\"\"\"\nSchema update migrations for the datastore.\n\nThis module contains all schema version upgrade methods (update_1 through update_N).\nThese are mixed into ChangeDetectionStore to keep the main store file focused.\n\nIMPORTANT: Each update could be run even when they have a new install and the schema is correct.\nTherefore - each `update_n` should be very careful about checking if it needs to actually run.\n\"\"\"\n\nimport os\nimport re\nimport shutil\nimport tarfile\nimport time\nfrom loguru import logger\nfrom copy import deepcopy\n\n\n# Try to import orjson for faster JSON serialization\ntry:\n    import orjson\n    HAS_ORJSON = True\nexcept ImportError:\n    HAS_ORJSON = False\n\nfrom ..html_tools import TRANSLATE_WHITESPACE_TABLE\nfrom ..processors.restock_diff import Restock\nfrom ..blueprint.rss import RSS_CONTENT_FORMAT_DEFAULT\nfrom ..model import USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH\n\ndef create_backup_tarball(datastore_path, update_number):\n    \"\"\"\n    Create a tarball backup of the entire datastore structure before running an update.\n\n    Includes:\n    - All {uuid}/watch.json files\n    - All {uuid}/tag.json files\n    - changedetection.json (settings, if it exists)\n    - url-watches.json (legacy format, if it exists)\n    - Directory structure preserved\n\n    Args:\n        datastore_path: Path to datastore directory\n        update_number: Update number being applied\n\n    Returns:\n        str: Path to created tarball, or None if backup failed\n\n    Restoration:\n    To restore from a backup:\n        cd /path/to/datastore\n        tar -xzf before-update-N-timestamp.tar.gz\n    This will restore all watch.json and tag.json files and settings to their pre-update state.\n    \"\"\"\n    timestamp = int(time.time())\n    backup_filename = f\"before-update-{update_number}-{timestamp}.tar.gz\"\n    backup_path = os.path.join(datastore_path, backup_filename)\n\n    try:\n        logger.info(f\"Creating backup tarball: {backup_filename}\")\n\n        with tarfile.open(backup_path, \"w:gz\") as tar:\n            # Backup changedetection.json if it exists (new format)\n            changedetection_json = os.path.join(datastore_path, \"changedetection.json\")\n            if os.path.isfile(changedetection_json):\n                tar.add(changedetection_json, arcname=\"changedetection.json\")\n                logger.debug(\"Added changedetection.json to backup\")\n\n            # Backup url-watches.json if it exists (legacy format)\n            url_watches_json = os.path.join(datastore_path, \"url-watches.json\")\n            if os.path.isfile(url_watches_json):\n                tar.add(url_watches_json, arcname=\"url-watches.json\")\n                logger.debug(\"Added url-watches.json to backup\")\n\n            # Backup all watch/tag directories with their JSON files\n            # This preserves the UUID directory structure\n            watch_count = 0\n            tag_count = 0\n            for entry in os.listdir(datastore_path):\n                entry_path = os.path.join(datastore_path, entry)\n\n                # Skip if not a directory\n                if not os.path.isdir(entry_path):\n                    continue\n\n                # Skip hidden directories and backup directories\n                if entry.startswith('.') or entry.startswith('before-update-'):\n                    continue\n\n                # Backup watch.json if exists\n                watch_json = os.path.join(entry_path, \"watch.json\")\n                if os.path.isfile(watch_json):\n                    tar.add(watch_json, arcname=f\"{entry}/watch.json\")\n                    watch_count += 1\n\n                    if watch_count % 100 == 0:\n                        logger.debug(f\"Backed up {watch_count} watch.json files...\")\n\n                # Backup tag.json if exists\n                tag_json = os.path.join(entry_path, \"tag.json\")\n                if os.path.isfile(tag_json):\n                    tar.add(tag_json, arcname=f\"{entry}/tag.json\")\n                    tag_count += 1\n\n            logger.success(f\"Backup created: {backup_filename} ({watch_count} watches from disk, {tag_count} tags from disk)\")\n            return backup_path\n\n    except Exception as e:\n        logger.error(f\"Failed to create backup tarball: {e}\")\n        # Try to clean up partial backup\n        if os.path.exists(backup_path):\n            try:\n                os.unlink(backup_path)\n            except:\n                pass\n        return None\n\n\nclass DatastoreUpdatesMixin:\n    \"\"\"\n    Mixin class containing all schema update methods.\n\n    This class is inherited by ChangeDetectionStore to provide schema migration functionality.\n    Each update_N method upgrades the schema from version N-1 to version N.\n    \"\"\"\n\n    def get_updates_available(self):\n        \"\"\"\n        Discover all available update methods.\n\n        Returns:\n            list: Sorted list of update version numbers (e.g., [1, 2, 3, ..., 26])\n        \"\"\"\n        import inspect\n        updates_available = []\n        for i, o in inspect.getmembers(self, predicate=inspect.ismethod):\n            m = re.search(r'update_(\\d+)$', i)\n            if m:\n                updates_available.append(int(m.group(1)))\n        updates_available.sort()\n\n        return updates_available\n\n    def run_updates(self, current_schema_version=None):\n        import sys\n        \"\"\"\n        Run all pending schema updates sequentially.\n\n        Args:\n            current_schema_version: Optional current schema version. If provided, only run updates\n                                   greater than this version. If None, uses the schema version from\n                                   the datastore. If no schema version exists in datastore and it appears\n                                   to be a fresh install, sets to latest update number (no updates needed).\n\n        IMPORTANT: Each update could be run even when they have a new install and the schema is correct.\n        Therefore - each `update_n` should be very careful about checking if it needs to actually run.\n\n        Process:\n        1. Get list of available updates\n        2. For each update > current schema version:\n           - Create backup of datastore\n           - Run update method\n           - Update schema version and commit settings\n           - Commit all watches and tags\n        3. If any update fails, stop processing\n        4. All changes saved via individual .commit() calls\n        \"\"\"\n        updates_available = self.get_updates_available()\n        if self.data.get('watching'):\n            test_watch = self.data['watching'].get(next(iter(self.data.get('watching', {}))))\n            from ..model.Watch import model\n\n            if not isinstance(test_watch, model):\n                import sys\n                logger.critical(\"Cannot run updates! Watch structure must be re-hydrated back to a Watch model object!\")\n                sys.exit(1)\n\n        if self.data['settings']['application'].get('tags',{}):\n            test_tag = self.data['settings']['application'].get('tags',{}).get(next(iter(self.data['settings']['application'].get('tags',{}))))\n            from ..model.Tag import model as tag_model\n\n            if not isinstance(test_tag, tag_model):\n                import sys\n                logger.critical(\"Cannot run updates! Watch tag/group structure must be re-hydrated back to a Tag model object!\")\n                sys.exit(1)\n\n        # Determine current schema version\n        if current_schema_version is None:\n            # Check if schema_version exists in datastore\n            current_schema_version = self.data['settings']['application'].get('schema_version')\n\n            if current_schema_version is None:\n                # No schema version found - could be a fresh install or very old datastore\n                # If this is a fresh/new config with no watches, assume it's up-to-date\n                # and set to latest update number (no updates needed)\n                if len(self.data['watching']) == 0:\n                    # Get the highest update number from available update methods\n                    latest_update = updates_available[-1] if updates_available else 0\n                    logger.info(f\"No schema version found and no watches exist - assuming fresh install, setting schema_version to {latest_update}\")\n                    self.data['settings']['application']['schema_version'] = latest_update\n                    self.commit()\n                    return  # No updates needed for fresh install\n                else:\n                    # Has watches but no schema version - likely old datastore, run all updates\n                    logger.warning(\"No schema version found but watches exist - running all updates from version 0\")\n                    current_schema_version = 0\n\n        logger.info(f\"Current schema version: {current_schema_version}\")\n\n        updates_ran = []\n\n        for update_n in updates_available:\n            if update_n > current_schema_version:\n                logger.critical(f\"Applying update_{update_n}\")\n\n                # Create tarball backup of entire datastore structure\n                # This includes all watch.json files, settings, and preserves directory structure\n                backup_path = create_backup_tarball(self.datastore_path, update_n)\n                if backup_path:\n                    logger.info(f\"Backup created at: {backup_path}\")\n                else:\n                    logger.warning(\"Backup creation failed, but continuing with update\")\n\n                try:\n                    update_method = getattr(self, f\"update_{update_n}\")()\n                except Exception as e:\n                    logger.critical(f\"Error while trying update_{update_n}\")\n                    logger.exception(e)\n                    sys.exit(1)\n                else:\n                    # Bump the version\n                    self.data['settings']['application']['schema_version'] = update_n\n                    self.commit()\n\n                    logger.success(f\"Update {update_n} completed\")\n\n                    # Track which updates ran\n                    updates_ran.append(update_n)\n\n    # ============================================================================\n    # Individual Update Methods\n    # ============================================================================\n\n    def update_1(self):\n        \"\"\"Convert minutes to seconds on settings and each watch.\"\"\"\n        if self.data['settings']['requests'].get('minutes_between_check'):\n            self.data['settings']['requests']['time_between_check']['minutes'] = self.data['settings']['requests']['minutes_between_check']\n            # Remove the default 'hours' that is set from the model\n            self.data['settings']['requests']['time_between_check']['hours'] = None\n\n        for uuid, watch in self.data['watching'].items():\n            if 'minutes_between_check' in watch:\n                # Only upgrade individual watch time if it was set\n                if watch.get('minutes_between_check', False):\n                    self.data['watching'][uuid]['time_between_check']['minutes'] = watch['minutes_between_check']\n\n    def update_2(self):\n        \"\"\"\n        Move the history list to a flat text file index.\n        Better than SQLite because this list is only appended to, and works across NAS / NFS type setups.\n        \"\"\"\n        # @todo test running this on a newly updated one (when this already ran)\n        for uuid, watch in self.data['watching'].items():\n            history = []\n\n            if watch.get('history', False):\n                for d, p in watch['history'].items():\n                    d = int(d)  # Used to be keyed as str, we'll fix this now too\n                    history.append(\"{},{}\\n\".format(d, p))\n\n                if len(history):\n                    target_path = os.path.join(self.datastore_path, uuid)\n                    if os.path.exists(target_path):\n                        with open(os.path.join(target_path, \"history.txt\"), \"w\") as f:\n                            f.writelines(history)\n                    else:\n                        logger.warning(f\"Datastore history directory {target_path} does not exist, skipping history import.\")\n\n                # No longer needed, dynamically pulled from the disk when needed.\n                # But we should set it back to a empty dict so we don't break if this schema runs on an earlier version.\n                # In the distant future we can remove this entirely\n                self.data['watching'][uuid]['history'] = {}\n\n    def update_3(self):\n        \"\"\"We incorrectly stored last_changed when there was not a change, and then confused the output list table.\"\"\"\n        # see https://github.com/dgtlmoon/changedetection.io/pull/835\n        return\n\n    def update_4(self):\n        \"\"\"`last_changed` not needed, we pull that information from the history.txt index.\"\"\"\n        for uuid, watch in self.data['watching'].items():\n            try:\n                # Remove it from the struct\n                del(watch['last_changed'])\n            except:\n                continue\n        return\n\n    def update_5(self):\n        \"\"\"\n        If the watch notification body, title look the same as the global one, unset it, so the watch defaults back to using the main settings.\n        In other words - the watch notification_title and notification_body are not needed if they are the same as the default one.\n        \"\"\"\n        current_system_body = self.data['settings']['application']['notification_body'].translate(TRANSLATE_WHITESPACE_TABLE)\n        current_system_title = self.data['settings']['application']['notification_body'].translate(TRANSLATE_WHITESPACE_TABLE)\n        for uuid, watch in self.data['watching'].items():\n            try:\n                watch_body = watch.get('notification_body', '')\n                if watch_body and watch_body.translate(TRANSLATE_WHITESPACE_TABLE) == current_system_body:\n                    # Looks the same as the default one, so unset it\n                    watch['notification_body'] = None\n\n                watch_title = watch.get('notification_title', '')\n                if watch_title and watch_title.translate(TRANSLATE_WHITESPACE_TABLE) == current_system_title:\n                    # Looks the same as the default one, so unset it\n                    watch['notification_title'] = None\n            except Exception as e:\n                continue\n        return\n\n    def update_7(self):\n        \"\"\"\n        We incorrectly used common header overrides that should only apply to Requests.\n        These are now handled in content_fetcher::html_requests and shouldnt be passed to Playwright/Selenium.\n        \"\"\"\n        # These were hard-coded in early versions\n        for v in ['User-Agent', 'Accept', 'Accept-Encoding', 'Accept-Language']:\n            if self.data['settings']['headers'].get(v):\n                del self.data['settings']['headers'][v]\n\n    def update_8(self):\n        \"\"\"Convert filters to a list of filters css_filter -> include_filters.\"\"\"\n        for uuid, watch in self.data['watching'].items():\n            try:\n                existing_filter = watch.get('css_filter', '')\n                if existing_filter:\n                    watch['include_filters'] = [existing_filter]\n            except:\n                continue\n        return\n\n    def update_9(self):\n        \"\"\"Convert old static notification tokens to jinja2 tokens.\"\"\"\n        # Each watch\n        # only { } not {{ or }}\n        r = r'(? '{key_exists_as_value}'\")\n                self.data['settings']['application']['notification_format'] = key_exists_as_value\n\n            for uuid, watch in self.data['watching'].items():\n                n_format = self.data['watching'][uuid].get('notification_format')\n                key_exists_as_value = next((k for k, v in formats.items() if v == n_format), None)\n                if key_exists_as_value and key_exists_as_value != USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH:  # key of \"Plain text\"\n                    logger.success(f\"['watching'][{uuid}]['notification_format'] '{n_format}' -> '{key_exists_as_value}'\")\n                    self.data['watching'][uuid]['notification_format'] = key_exists_as_value  # should be 'text' or whatever\n\n            for uuid, tag in self.data['settings']['application']['tags'].items():\n                n_format = self.data['settings']['application']['tags'][uuid].get('notification_format')\n                key_exists_as_value = next((k for k, v in formats.items() if v == n_format), None)\n                if key_exists_as_value and key_exists_as_value != USE_SYSTEM_DEFAULT_NOTIFICATION_FORMAT_FOR_WATCH:  # key of \"Plain text\"\n                    logger.success(\n                        f\"['settings']['application']['tags'][{uuid}]['notification_format'] '{n_format}' -> '{key_exists_as_value}'\")\n                    self.data['settings']['application']['tags'][uuid][\n                        'notification_format'] = key_exists_as_value  # should be 'text' or whatever\n\n        from ..notification import valid_notification_formats\n        formats = deepcopy(valid_notification_formats)\n        re_run(formats)\n        # And in previous versions, it was \"text\" instead of Plain text, Markdown instead of \"Markdown to HTML\"\n        formats['text'] = 'Text'\n        formats['markdown'] = 'Markdown'\n        re_run(formats)\n\n    def update_24(self):\n        \"\"\"RSS types should be inline with the same names as notification types.\"\"\"\n        rss_format = self.data['settings']['application'].get('rss_content_format')\n        if not rss_format or 'text' in rss_format:\n            # might have been 'plaintext, 'plain text' or something\n            self.data['settings']['application']['rss_content_format'] = RSS_CONTENT_FORMAT_DEFAULT\n        elif 'html' in rss_format:\n            self.data['settings']['application']['rss_content_format'] = 'htmlcolor'\n        else:\n            # safe fallback to text\n            self.data['settings']['application']['rss_content_format'] = RSS_CONTENT_FORMAT_DEFAULT\n\n    def update_25(self):\n        \"\"\"Different processors now hold their own history.txt.\"\"\"\n        for uuid, watch in self.data['watching'].items():\n            processor = self.data['watching'][uuid].get('processor')\n            if processor != 'text_json_diff':\n                old_history_txt = os.path.join(self.datastore_path, \"history.txt\")\n                target_history_name = f\"history-{processor}.txt\"\n                if os.path.isfile(old_history_txt) and not os.path.isfile(target_history_name):\n                    new_history_txt = os.path.join(self.datastore_path, target_history_name)\n                    logger.debug(f\"Renaming history index {old_history_txt} to {new_history_txt}...\")\n                    shutil.move(old_history_txt, new_history_txt)\n\n    def migrate_legacy_db_format(self):\n        \"\"\"\n        Migration: Individual watch persistence (COPY-based, safe rollback).\n\n        Loads legacy url-watches.json format and migrates to:\n        - {uuid}/watch.json (per watch)\n        - changedetection.json (settings only)\n\n        IMPORTANT:\n        - A tarball backup (before-update-26-timestamp.tar.gz) is created before migration\n        - url-watches.json is LEFT INTACT for rollback safety\n        - Users can roll back by simply downgrading to the previous version\n        - Or restore from tarball: tar -xzf before-update-26-*.tar.gz\n\n        This is a dedicated migration release - users upgrade at their own pace.\n        \"\"\"\n        logger.critical(\"=\" * 80)\n        logger.critical(\"Running migration: Individual watch persistence (update_26)\")\n        logger.critical(\"COPY-based migration: url-watches.json will remain intact for rollback\")\n        logger.critical(\"=\" * 80)\n\n        # Populate settings from legacy data\n        logger.info(\"Populating settings from legacy data...\")\n        watch_count = len(self.data['watching'])\n        logger.success(f\"Loaded {watch_count} watches from legacy format\")\n\n        # Phase 1: Save all watches to individual files\n        logger.critical(f\"Phase 1/4: Saving {watch_count} watches to individual watch.json files...\")\n\n        saved_count = 0\n        for uuid, watch in self.data['watching'].items():\n            try:\n                watch.commit()\n                saved_count += 1\n\n                if saved_count % 100 == 0:\n                    logger.info(f\"  Progress: {saved_count}/{watch_count} watches migrated...\")\n\n            except Exception as e:\n                logger.error(f\"Failed to save watch {uuid}: {e}\")\n                raise Exception(\n                    f\"Migration failed: Could not save watch {uuid}. \"\n                    f\"url-watches.json remains intact, safe to retry. Error: {e}\"\n                )\n\n        logger.critical(f\"Phase 1 complete: Saved {saved_count} watches\")\n\n        # Phase 2: Verify all files exist\n        logger.critical(\"Phase 2/4: Verifying all watch.json files were created...\")\n\n        missing = []\n        for uuid in self.data['watching'].keys():\n            watch_json = os.path.join(self.datastore_path, uuid, \"watch.json\")\n            if not os.path.isfile(watch_json):\n                missing.append(uuid)\n\n        if missing:\n            raise Exception(\n                f\"Migration failed: {len(missing)} watch files missing: {missing[:5]}... \"\n                f\"url-watches.json remains intact, safe to retry.\"\n            )\n\n        logger.critical(f\"Phase 2 complete: Verified {watch_count} watch files\")\n\n        # Phase 3: Create new settings file\n        logger.critical(\"Phase 3/4: Creating changedetection.json...\")\n\n        try:\n            self._save_settings()\n        except Exception as e:\n            logger.error(f\"Failed to create changedetection.json: {e}\")\n            raise Exception(\n                f\"Migration failed: Could not create changedetection.json. \"\n                f\"url-watches.json remains intact, safe to retry. Error: {e}\"\n            )\n\n        # Phase 4: Verify settings file exists\n        logger.critical(\"Phase 4/4: Verifying changedetection.json exists...\")\n        changedetection_json_new_schema=os.path.join(self.datastore_path, \"changedetection.json\")\n        if not os.path.isfile(changedetection_json_new_schema):\n            import sys\n            logger.critical(\"Migration failed, changedetection.json not found after update ran!\")\n            sys.exit(1)\n\n\n        logger.critical(\"Phase 4 complete: Verified changedetection.json exists\")\n\n        # Success! Now reload from new format\n        logger.critical(\"Reloading datastore from new format...\")\n        # write it to disk, it will be saved without ['watching'] in the JSON db because we find it from disk glob\n        self._save_settings()\n        logger.success(\"Datastore reloaded from new format successfully\")\n        logger.critical(\"=\" * 80)\n        logger.critical(\"MIGRATION COMPLETED SUCCESSFULLY!\")\n        logger.critical(\"=\" * 80)\n        logger.info(\"\")\n        logger.info(\"New format:\")\n        logger.info(f\"  - {watch_count} individual watch.json files created\")\n        logger.info(f\"  - changedetection.json created (settings only)\")\n        logger.info(\"\")\n        logger.info(\"Rollback safety:\")\n        logger.info(\"  - url-watches.json preserved for rollback\")\n        logger.info(\"  - To rollback: downgrade to previous version and restart\")\n        logger.info(\"  - No manual file operations needed\")\n        logger.info(\"\")\n        logger.info(\"Optional cleanup (after testing new version):\")\n        logger.info(f\"  - rm {os.path.join(self.datastore_path, 'url-watches.json')}\")\n        logger.info(\"\")\n\n    def update_26(self):\n        self.migrate_legacy_db_format()\n\n    # Re-run tag to JSON migration\n    def update_29(self):\n\n        \"\"\"\n        Migrate tags to individual tag.json files.\n\n        Tags are currently saved only in changedetection.json (settings).\n        This migration ALSO saves them to individual {uuid}/tag.json files,\n        similar to how watches are stored (dual storage).\n\n        Benefits:\n        - Allows atomic tag updates without rewriting entire settings\n        - Enables independent tag versioning/backup\n        - Maintains backwards compatibility (tags stay in settings too)\n        \"\"\"\n        logger.critical(\"=\" * 80)\n        logger.critical(\"Running migration: Individual tag persistence (update_28)\")\n        logger.critical(\"Creating individual tag.json files\")\n        logger.critical(\"=\" * 80)\n\n        tags = self.data['settings']['application'].get('tags', {})\n        tag_count = len(tags)\n\n        if tag_count == 0:\n            logger.info(\"No tags found, skipping migration\")\n            return\n\n        logger.info(f\"Migrating {tag_count} tags to individual tag.json files...\")\n\n        saved_count = 0\n        failed_count = 0\n\n        for uuid, tag_data in tags.items():\n            if os.path.isfile(os.path.join(self.datastore_path, uuid, \"tag.json\")):\n                logger.debug(f\"Tag {uuid} tag.json exists, skipping\")\n                continue\n            try:\n                tag_data.commit()\n                saved_count += 1\n                if saved_count % 10 == 0:\n                    logger.info(f\"  Progress: {saved_count}/{tag_count} tags migrated...\")\n\n            except Exception as e:\n                logger.error(f\"Failed to save tag {uuid} ({tag_data.get('title', 'unknown')}): {e}\")\n                failed_count += 1\n\n        if failed_count > 0:\n            logger.warning(f\"Migration complete: {saved_count} tags saved, {failed_count} tags FAILED\")\n        else:\n            logger.success(f\"Migration complete: {saved_count} tags saved to individual tag.json files\")\n\n        # Tags remain in settings for backwards compatibility AND easy access\n        # On next load, _load_tags() will read from tag.json files and merge with settings\n        logger.info(\"Tags saved to both settings AND individual tag.json files\")\n        logger.info(\"Future tag edits will update both locations (dual storage)\")\n        logger.critical(\"=\" * 80)\n\n        # write it to disk, it will be saved without ['tags'] in the JSON db because we find it from disk glob\n        # (left this out by accident in previous update, added tags={} in the changedetection.json save_to_disk)\n        self._save_settings()\n\n    def update_30(self):\n        \"\"\"Migrate restock_settings out of watch.json into restock_diff.json processor config file.\n\n        Previously, restock_diff processor settings (in_stock_processing, follow_price_changes, etc.)\n        were stored directly in the watch dict (watch.json). They now belong in a separate per-watch\n        processor config file (restock_diff.json) consistent with the processor_config_* API system.\n\n        For tags: restock_settings key is renamed to processor_config_restock_diff in the tag dict,\n        matching what the API writes when updating a tag.\n\n        Safe to re-run: skips watches that already have a restock_diff.json, skips tags that already\n        have processor_config_restock_diff set.\n        \"\"\"\n        import json\n\n        # --- Watches ---\n        for uuid, watch in self.data['watching'].items():\n            if watch.get('processor') != 'restock_diff':\n                continue\n            restock_settings = watch.get('restock_settings')\n            if not restock_settings:\n                continue\n\n            data_dir = watch.data_dir\n            if data_dir:\n                watch.ensure_data_dir_exists()\n                filepath = os.path.join(data_dir, 'restock_diff.json')\n                if not os.path.isfile(filepath):\n                    with open(filepath, 'w', encoding='utf-8') as f:\n                        json.dump({'restock_diff': restock_settings}, f, indent=2)\n                    logger.info(f\"update_30: migrated restock_settings → {filepath}\")\n\n            del self.data['watching'][uuid]['restock_settings']\n            watch.commit()\n\n        # --- Tags ---\n        for tag_uuid, tag in self.data['settings']['application']['tags'].items():\n            restock_settings = tag.get('restock_settings')\n            if not restock_settings or tag.get('processor_config_restock_diff'):\n                continue\n            tag['processor_config_restock_diff'] = restock_settings\n            del tag['restock_settings']\n            tag.commit()\n            logger.info(f\"update_30: migrated tag {tag_uuid} restock_settings → processor_config_restock_diff\")\n\n"
  },
  {
    "path": "changedetectionio/strtobool.py",
    "content": "# Because strtobool was removed in python 3.12 distutils\n\n_MAP = {\n    'y': True,\n    'yes': True,\n    't': True,\n    'true': True,\n    'on': True,\n    '1': True,\n    'n': False,\n    'no': False,\n    'f': False,\n    'false': False,\n    'off': False,\n    '0': False\n}\n\n\ndef strtobool(value):\n    if not value:\n        return False\n    try:\n        return _MAP[str(value).lower()]\n    except KeyError:\n        raise ValueError('\"{}\" is not a valid bool value'.format(value))\n"
  },
  {
    "path": "changedetectionio/templates/IMPORTANT.md",
    "content": "# Important notes about templates\n\nTemplate names should always end in \".html\", \".htm\", \".xml\", \".xhtml\", \".svg\", even the `import`'ed templates.\n\nJinja2's `def select_jinja_autoescape(self, filename: str) -> bool:` will check the filename extension and enable autoescaping\n\n"
  },
  {
    "path": "changedetectionio/templates/_common_fields.html",
    "content": "\n{% from '_helpers.html' import render_field %}\n\n{% macro show_token_placeholders(extra_notification_token_placeholder_info, suffix=\"\") %}\n\n\n    
\n            \n        {{ _('Body for all notifications — You can use') }} Jinja2 {{ _('templating in the notification title, body and URL, and tokens from below.') }}\n    
\n        
{{ _('Show token/placeholders') }}\n        
\n    
\n    
\n        \n            \n            \n                \n                \n            \n            \n            \n            \n                \n                \n            \n            \n                \n                \n            \n            \n                \n                \n            \n            \n                \n                ' in res.data\n    delete_all_watches(client)\n\n# If there was only a change in the whitespacing, then we shouldnt have a change detected\ndef test_lev_conditions_plugin(client, live_server, measure_memory_usage, datastore_path):\n    # This should break..\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(\"\"\"\n       \n     Some initial text
\n     
Which is across multiple lines
\n     
\n     So let's see what happens.  
\n     \n     \n    \"\"\")\n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True)\n    res = client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": test_url, \"tags\": '', 'edit_and_watch_submit_button': 'Edit > Watch'},\n        follow_redirects=True\n    )\n    assert b\"Watch added in Paused state, saving will unpause\" in res.data\n\n    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))\n    # Give the thread time to pick it up\n    wait_for_all_checks(client)\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=uuid, unpause_on_save=1),\n        data={\n            \"url\": test_url,\n            \"fetch_backend\": \"html_requests\",\n            \"conditions_match_logic\": CONDITIONS_MATCH_LOGIC_DEFAULT,  # ALL = AND logic\n            \"conditions-0-field\": \"levenshtein_ratio\",\n            \"conditions-0-operator\": \"<\",\n            \"conditions-0-value\": \"0.8\", # needs to be more of a diff to trigger a change\n            \"time_between_check_use_default\": \"y\"\n        },\n        follow_redirects=True\n    )\n\n    assert b\"unpaused\" in res.data\n\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'has-unread-changes' not in res.data\n\n    # Check the content saved initially, even tho a condition was set - this is the first snapshot so shouldnt be affected by conditions\n    res = client.get(\n        url_for(\"ui.ui_preview.preview_page\", uuid=uuid),\n        follow_redirects=True\n    )\n    assert b'Which is across multiple lines' in res.data\n\n\n    ############### Now change it a LITTLE bit...\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(\"\"\"\n       \n     Some initial text
\n     
Which is across multiple lines
\n     
\n     So let's see what happenxxxxxxxxx.  
\n     \n     \n    \"\"\")\n\n    res = client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    assert b'Queued 1 watch for rechecking.' in res.data\n    wait_for_all_checks(client)\n\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'has-unread-changes' not in res.data #because this will be like 0.90 not 0.8 threshold\n\n    ############### Now change it a MORE THAN 50%\n    test_return_data = \"\"\"\n       \n     Some sxxxx
\n     
Which is across a lines
\n     
\n     ok.  
\n     \n     \n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n    res = client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    assert b'Queued 1 watch for rechecking.' in res.data\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'has-unread-changes' in res.data\n    # cleanup for the next\n    client.get(\n        url_for(\"ui.form_delete\", uuid=\"all\"),\n        follow_redirects=True\n    )"
  },
  {
    "path": "changedetectionio/tests/test_css_selector.py",
    "content": "#!/usr/bin/env python3\n\nimport time\nfrom flask import url_for\nfrom .util import live_server_setup, wait_for_all_checks\nimport os\n\nfrom ..html_tools import *\n\n\n\ndef set_original_response(datastore_path):\n    test_return_data = \"\"\"\n       \n     Some initial text
\n     
Which is across multiple lines
\n     
\n     So let's see what happens.  
\n     
Some text thats the same
\n     
Some text that will change
\n     \n     \n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n    return None\n\ndef set_modified_response(datastore_path):\n    test_return_data = \"\"\"\n       \n     Some initial text
\n     
which has this one new line
\n     
\n     So let's see what happens.  
\n     
Some text thats the same
\n     
Some text that changes
\n     \n     \n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n\n    return None\n\n\n# Test that the CSS extraction works how we expect, important here is the right placing of new lines \\n's\ndef test_include_filters_output():\n    from inscriptis import get_text\n\n    # Check text with sub-parts renders correctly\n    content = \"\"\"
  Some really bold text  
\"\"\"\n    html_blob = include_filters(include_filters=\"#thingthing\", html_content=content)\n    text = get_text(html_blob)\n    assert text == \"  Some really bold text\"\n\n    content = \"\"\"\n    
foo bar blah
\n    
Block A
Block B
 \n    \n\"\"\"\n\n    # in xPath this would be //*[@class='parts']\n    html_blob = include_filters(include_filters=\".parts\", html_content=content)\n    text = get_text(html_blob)\n\n    # Divs are converted to 4 whitespaces by inscriptis\n    assert text == \"    Block A\\n    Block B\"\n\n\n# Tests the whole stack works with the CSS Filter\ndef test_check_markup_include_filters_restriction(client, live_server, measure_memory_usage, datastore_path):\n\n    include_filters = \"#sametext\"\n\n    set_original_response(datastore_path=datastore_path)\n\n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    wait_for_all_checks(client)\n\n\n    # Goto the edit page, add our ignore text\n    # Add our URL to the import page\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={\"include_filters\": include_filters, \"url\": test_url, \"tags\": \"\", \"headers\": \"\", 'fetch_backend': \"html_requests\", \"time_between_check_use_default\": \"y\"},\n        follow_redirects=True\n    )\n    assert b\"Updated watch.\" in res.data\n    time.sleep(1)\n    # Check it saved\n    res = client.get(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n    )\n    assert bytes(include_filters.encode('utf-8')) in res.data\n\n    wait_for_all_checks(client)\n\n    #  Make a change\n    set_modified_response(datastore_path=datastore_path)\n\n    # Trigger a check\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n\n    # It should have 'has-unread-changes' still\n    # Because it should be looking at only that 'sametext' id\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'has-unread-changes' in res.data\n\n\n# Tests the whole stack works with the CSS Filter\ndef test_check_multiple_filters(client, live_server, measure_memory_usage, datastore_path):\n    \n    include_filters = \"#blob-a\\r\\nxpath://*[contains(@id,'blob-b')]\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(\"\"\"\n     
Blob A
\n     
Blob B
\n     
Blob C
\n     \n     \n    \"\"\")\n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    # Goto the edit page, add our ignore text\n    # Add our URL to the import page\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={\"include_filters\": include_filters,\n              \"url\": test_url,\n              \"tags\": \"\",\n              \"headers\": \"\",\n              'fetch_backend': \"html_requests\",\n              \"time_between_check_use_default\": \"y\"},\n        follow_redirects=True\n    )\n    assert b\"Updated watch.\" in res.data\n\n    # Give the thread time to pick it up\n    wait_for_all_checks(client)\n\n    res = client.get(\n        url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"),\n        follow_redirects=True\n    )\n\n    # Only the two blobs should be here\n    assert b\"Blob A\" in res.data # CSS was ok\n    assert b\"Blob B\" in res.data # xPath was ok\n    assert b\"Blob C\" not in res.data # Should not be included\n\n# The filter exists, but did not contain anything useful\n# Mainly used when the filter contains just an IMG, this can happen when someone selects an image in the visual-selector\n# Tests fetcher can throw a \"ReplyWithContentButNoText\" exception after applying filter and extracting text\ndef test_filter_is_empty_help_suggestion(client, live_server, measure_memory_usage, datastore_path):\n    \n\n    include_filters = \"#blob-a\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(\"\"\"\n         
\n           \n         
\n         \n         \n        \"\"\")\n\n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    # Goto the edit page, add our ignore text\n    # Add our URL to the import page\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={\"include_filters\": include_filters,\n              \"url\": test_url,\n              \"tags\": \"\",\n              \"headers\": \"\",\n              'fetch_backend': \"html_requests\",\n              \"time_between_check_use_default\": \"y\"},\n        follow_redirects=True\n    )\n    assert b\"Updated watch.\" in res.data\n\n    wait_for_all_checks(client)\n\n\n    res = client.get(\n        url_for(\"watchlist.index\"),\n        follow_redirects=True\n    )\n\n    assert b'empty result or contain only an image' in res.data\n\n\n    ### Just an empty selector, no image\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(\"\"\"\n         
\n           \n         
\n         \n         \n        \"\"\")\n\n    res = client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    res = client.get(\n        url_for(\"watchlist.index\"),\n        follow_redirects=True\n    )\n\n    assert b'empty result or contain only an image' not in res.data\n    assert b'but contained no usable text' in res.data\n"
  },
  {
    "path": "changedetectionio/tests/test_datastore_isolation.py",
    "content": "#!/usr/bin/env python3\n\"\"\"Test to verify client and live_server share the same datastore\"\"\"\n\ndef test_client_and_live_server_share_datastore(client, live_server):\n    \"\"\"Verify that client and live_server use the same app and datastore.\"\"\"\n\n    # They should be the SAME object\n    assert client.application is live_server.app, \"client.application and live_server.app should be the SAME object!\"\n\n    # They should share the same datastore\n    client_datastore = client.application.config.get('DATASTORE')\n    server_datastore = live_server.app.config.get('DATASTORE')\n\n    assert client_datastore is server_datastore, \\\n        f\"Datastores are DIFFERENT objects! client={hex(id(client_datastore))} server={hex(id(server_datastore))}\"\n\n    print(f\"✓ client.application and live_server.app are the SAME object\")\n    print(f\"✓ Both use the same DATASTORE at {hex(id(client_datastore))}\")\n    print(f\"✓ Datastore path: {client_datastore.datastore_path}\")\n"
  },
  {
    "path": "changedetectionio/tests/test_element_removal.py",
    "content": "#!/usr/bin/env python3\n\nimport time\nimport os\n\nfrom flask import url_for\n\nfrom ..html_tools import *\nfrom .util import live_server_setup, wait_for_all_checks, delete_all_watches\n\n\n\n\ndef set_response_with_multiple_index(datastore_path):\n    data= \"\"\"\n\n\n\n\n
{{ _('Token') }}{{ _('Description') }}
{{ '{{base_url}}' }}{{ _('The URL of the changedetection.io instance you are running.') }}
{{ '{{watch_url}}' }}{{ _('The URL being watched.') }}
{{ '{{watch_uuid}}' }}{{ _('The UUID of the watch.') }}
{{ '{{watch_title}}' }}{{ _('The page title of the watch, uses  if not set, falls back to URL') }}</td>\n            </tr>\n            <tr>\n                <td><code>{{ '{{watch_tag}}' }}</code></td>\n                <td>{{ _('The watch group / tag') }}</td>\n            </tr>\n            <tr>\n                <td><code>{{ '{{preview_url}}' }}</code></td>\n                <td>{{ _('The URL of the preview page generated by changedetection.io.') }}</td>\n            </tr>\n            <tr>\n                <td><code>{{ '{{change_datetime}}' }}</code></td>\n                <td>{{ _('Date/time of the change, accepts format=, change_datetime(format=\\'%A\\')\\', default is \\'%Y-%m-%d %H:%M:%S %Z\\'') }}</td>\n            </tr>\n            <tr>\n                <td><code>{{ '{{diff_url}}' }}</code></td>\n                <td>{{ _('The URL of the diff output for the watch.') }}</td>\n            </tr>\n            <tr>\n                <td><code>{{ '{{diff_url}}' }}</code></td>\n                <td>{{ _('The URL of the diff output for the watch.') }}</td>\n            </tr>\n            <tr>\n                <td><code>{{ '{{diff}}' }}</code></td>\n                <td>{{ _('The diff output - only changes, additions, and removals') }}<br>\n                    <small>\n                        {{ _('All diff variants accept') }} <code>lines=</code>, <code>context=</code>, <code>word_diff=</code>, <code>ignore_junk=</code> {{ _('args, e.g.') }}\n                        <code>{{ '{{diff(lines=10)}}' }}</code>, <code>{{ '{{diff_added(lines=5, context=2)}}' }}</code>\n                    </small>\n                </td>\n            </tr>\n            <tr>\n                <td><code>{{ '{{diff_clean}}' }}</code></td>\n                <td>{{ _('The diff output - only changes, additions, and removals —') }} <i>{{ _('Without (added) prefix or colors') }}</i>\n                </td>\n            </tr>\n            <tr>\n                <td><code>{{ '{{diff_added}}' }}</code></td>\n                <td>{{ _('The diff output - only changes and additions') }}</td>\n            </tr>\n            <tr>\n                <td><code>{{ '{{diff_added_clean}}' }}</code></td>\n                <td>{{ _('The diff output - only changes and additions —') }} <i>{{ _('Without (added) prefix or colors') }}</i></td>\n            </tr>\n            <tr>\n                <td><code>{{ '{{diff_removed}}' }}</code></td>\n                <td>{{ _('The diff output - only changes and removals') }}</td>\n            </tr>\n            <tr>\n                <td><code>{{ '{{diff_removed_clean}}' }}</code></td>\n                <td>{{ _('The diff output - only changes and removals —') }} <i>{{ _('Without (added) prefix or colors') }}</i></td>\n            </tr>\n            <tr>\n                <td><code>{{ '{{diff_full}}' }}</code></td>\n                <td>{{ _('The diff output - full difference output') }}</td>\n            </tr>\n            <tr>\n                <td><code>{{ '{{diff_full_clean}}' }}</code></td>\n                <td>{{ _('The diff output - full difference output —') }} <i>{{ _('Without (added) prefix or colors') }}</i></td>\n            </tr>\n            <tr>\n                <td><code>{{ '{{diff_patch}}' }}</code></td>\n                <td>{{ _('The diff output - patch in unified format') }}</td>\n            </tr>\n            <tr>\n                <td><code>{{ '{{current_snapshot}}' }}</code></td>\n                <td>{{ _('The current snapshot text contents value, useful when combined with JSON or CSS filters') }}\n                </td>\n            </tr>\n            <tr>\n                <td><code>{{ '{{triggered_text}}' }}</code></td>\n                <td>{{ _('Text that tripped the trigger from filters') }}</td>\n\n                {% if extra_notification_token_placeholder_info %}\n                    {% for token in extra_notification_token_placeholder_info %}\n                        <tr>\n                            <td><code>{{ '{{' }}{{ token[0] }}{{ '}}' }}</code></td>\n                            <td>{{ token[1] }}</td>\n                        </tr>\n                    {% endfor %}\n                {% endif %}\n            </tbody>\n        </table>\n\n        <span class=\"pure-form-message-inline\">\n        {{ _('Warning: Contents of') }} <code>{{ '{{diff}}' }}</code>, <code>{{ '{{diff_removed}}' }}</code>, {{ _('and') }} <code>{{ '{{diff_added}}' }}</code> {{ _('depend on how the difference algorithm perceives the change.') }} <br>\n        {{ _('For example, an addition or removal could be perceived as a change in some cases.') }} <a target=\"newwindow\" href=\"https://github.com/dgtlmoon/changedetection.io/wiki/Using-the-%7B%7Bdiff%7D%7D,-%7B%7Bdiff_added%7D%7D,-and-%7B%7Bdiff_removed%7D%7D-notification-tokens\">{{ _('More Here') }}</a> <br>\n        </span>\n    </div>\n{%  endmacro %}\n\n{% macro render_common_settings_form(form, emailprefix, settings_application, extra_notification_token_placeholder_info) %}\n                        <div class=\"pure-control-group\">\n                            {{ render_field(form.notification_urls, rows=5, placeholder=\"Examples:\n    Gitter - gitter://token/room\n    Office365 - o365://TenantID:AccountEmail/ClientID/ClientSecret/TargetEmail\n    AWS SNS - sns://AccessKeyID/AccessSecretKey/RegionName/+PhoneNo\n    SMTPS - mailtos://user:pass@mail.domain.com?to=receivingAddress@example.com\",\n    class=\"notification-urls\" )\n                            }}\n                            <div class=\"pure-form-message-inline\">\n                                <p>\n                                <strong>{{ _('Tip:') }}</strong> {{ _('Use') }} <a target=\"newwindow\" href=\"https://github.com/caronc/apprise\">{{ _('AppRise Notification URLs') }}</a> {{ _('for notification to just about any service!') }} <i><a target=\"newwindow\" href=\"https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes\">{{ _('Please read the notification services wiki here for important configuration notes') }}</a></i>.<br>\n</p>\n                                <div data-target=\"#advanced-help-notifications\" class=\"toggle-show pure-button button-tag button-xsmall\">{{ _('Show advanced help and tips') }}</div>\n                                <ul style=\"display: none\" id=\"advanced-help-notifications\">\n                                <li><code><a target=\"newwindow\" href=\"https://github.com/caronc/apprise/wiki/Notify_discord\">discord://</a></code> {{ _('(or') }} <code>https://discord.com/api/webhooks...</code>)) {{ _('only supports a maximum') }} <strong>{{ _('2,000 characters') }}</strong> {{ _('of notification text, including the title.') }}</li>\n                                <li><code><a target=\"newwindow\" href=\"https://github.com/caronc/apprise/wiki/Notify_telegram\">tgram://</a></code> {{ _('bots can\\'t send messages to other bots, so you should specify chat ID of non-bot user.') }}</li>\n                                <li><code><a target=\"newwindow\" href=\"https://github.com/caronc/apprise/wiki/Notify_telegram\">tgram://</a></code> {{ _('only supports very limited HTML and can fail when extra tags are sent,') }} <a href=\"https://core.telegram.org/bots/api#html-style\">{{ _('read more here') }}</a> {{ _('(or use plaintext/markdown format)') }}</li>\n                                <li><code>gets://</code>, <code>posts://</code>, <code>puts://</code>, <code>deletes://</code> {{ _('for direct API calls (or omit the') }} \"<code>s</code>\" {{ _('for non-SSL ie') }} <code>get://</code>) <a href=\"https://github.com/dgtlmoon/changedetection.io/wiki/Notification-configuration-notes#postposts\">{{ _('more help here') }}</a></li>\n                                  <li>{{ _('Accepts the') }} <code>{{ '{{token}}' }}</code> {{ _('placeholders listed below') }}</li>\n                              </ul>\n                            </div>\n                            <div class=\"notifications-wrapper\">\n                              <a id=\"send-test-notification\" class=\"pure-button button-secondary button-xsmall\" >{{ _('Send test notification') }}</a> <div class=\"spinner\"  style=\"display: none;\"></div>\n                            {% if emailprefix %}\n                              <a id=\"add-email-helper\" class=\"pure-button button-secondary button-xsmall\" >{{ _('Add email') }} <img style=\"height: 1em; display: inline-block\" src=\"{{url_for('static_content', group='images', filename='email.svg')}}\" alt=\"{{ _('Add an email address') }}\"> </a>\n                            {% endif %}\n                              <a href=\"{{url_for('settings.notification_logs')}}\" class=\"pure-button button-secondary button-xsmall\" >{{ _('Notification debug logs') }}</a>\n                              <br>\n                                <div id=\"notification-test-log\" style=\"display: none;\"><span class=\"pure-form-message-inline\">{{ _('Processing..') }}</span></div>\n                            </div>\n                        </div>\n\n                        <div class=\"pure-control-group grey-form-border\">\n                            <div class=\"pure-control-group\">\n                                {{ render_field(form.notification_title, class=\"m-d notification-title\", placeholder=settings_application['notification_title']) }}\n                                <span class=\"pure-form-message-inline\">{{ _('Title for all notifications') }}</span>\n                            </div>\n                            <div class=\"pure-control-group\">\n                                {{ render_field(form.notification_body , rows=5, class=\"notification-body\", placeholder=settings_application['notification_body']) }}\n                                {{ show_token_placeholders(extra_notification_token_placeholder_info=extra_notification_token_placeholder_info) }}\n                                <div class=\"pure-form-message-inline\">\n                                    <ul>\n                                    <li><span class=\"pure-form-message-inline\">\n                                        {{ _('For JSON payloads, use') }} <strong>|tojson</strong> {{ _('without quotes for automatic escaping, for example -') }} <code>{ \"name\": {{ '{{ watch_title|tojson }}' }} }</code>\n                                    </span></li>\n                                    <li><span class=\"pure-form-message-inline\">\n                                        {{ _('URL encoding, use') }} <strong>|urlencode</strong>, {{ _('for example -') }} <code>gets://hook-website.com/test.php?title={{ '{{ watch_title|urlencode }}' }}</code>\n                                    </span></li>\n                                    <li><span class=\"pure-form-message-inline\">\n                                        {{ _('Regular-expression replace, use') }} <strong>|regex_replace</strong>, {{ _('for example -') }}   <code>{{ \"{{ \\\"hello world 123\\\" | regex_replace('[0-9]+', 'no-more-numbers') }}\" }}</code>\n                                    </span></li>\n                                    <li><span class=\"pure-form-message-inline\">\n                                        {{ _('For a complete reference of all Jinja2 built-in filters, users can refer to the') }} <a href=\"https://jinja.palletsprojects.com/en/3.1.x/templates/#builtin-filters\">https://jinja.palletsprojects.com/en/3.1.x/templates/#builtin-filters</a>\n                                    </span></li>\n                                    </ul>\n                                    <br>\n                                </div>\n                            </div>\n                            <div class=\"\">\n                                {{ render_field(form.notification_format , class=\"notification-format\") }}\n                                <span class=\"pure-form-message-inline\">{{ _('Format for all notifications') }}</span>\n                            </div>\n                        </div>\n{% endmacro %}\n"
  },
  {
    "path": "changedetectionio/templates/_helpers.html",
    "content": "{% macro render_field(field) %}\n    <div {% if field.errors or field.top_errors %} class=\"error\" {% endif %}>{{ field.label }}</div>\n    <div {% if field.errors or field.top_errors %} class=\"error\" {% endif %}>{{ field(**kwargs)|safe }}\n        {% if field.top_errors %}\n            top\n            <ul class=\"errors top-errors\">\n                {% for error in field.top_errors %}\n                    <li>{{ error }}</li>\n                {% endfor %}\n            </ul>\n        {% endif %}\n        {% if field.errors %}\n            <ul class=errors>\n                {% if field.errors is mapping and 'form' in field.errors %}\n                    {#  and subfield form errors, such as used in RequiredFormField() for TimeBetweenCheckForm sub form #}\n                    {% set errors = field.errors['form'] %}\n                    {% for error in errors %}\n                        <li>{{ error }}</li>\n                    {% endfor %}\n                {% elif field.type == 'FieldList' %}\n                    {# Handle FieldList of FormFields - errors is a list of dicts, one per entry #}\n                    {% for idx, entry_errors in field.errors|enumerate %}\n                        {% if entry_errors is mapping and entry_errors %}\n                            {# Only show entries that have actual errors #}\n                            <li><strong>{{ _('Entry') }} {{ idx + 1 }}:</strong>\n                                <ul>\n                                    {% for field_name, messages in entry_errors.items() %}\n                                        {% for message in messages %}\n                                            <li>{{ field_name }}: {{ message }}</li>\n                                        {% endfor %}\n                                    {% endfor %}\n                                </ul>\n                            </li>\n                        {% endif %}\n                    {% endfor %}\n                {% else %}\n                    {#  regular list of errors with this field #}\n                    {% for error in field.errors %}\n                        <li>{{ error }}</li>\n                    {% endfor %}\n                {% endif %}\n            </ul>\n        {% endif %}\n    </div>\n{% endmacro %}\n\n{% macro render_checkbox_field(field) %}\n  <div class=\"checkbox {% if field.errors %} error {% endif %}\">\n  {{ field(**kwargs)|safe }} <label for=\"{{ field.id }}\">{{ field.label.text | string | forceescape }}</label>\n  {% if field.errors %}\n    <ul class=errors>\n    {% for error in field.errors %}\n      <li>{{ error }}</li>\n    {% endfor %}\n    </ul>\n  {% endif %}\n  </div>\n{% endmacro %}\n\n{% macro render_ternary_field(field, BooleanField=false) %}\n  {% if BooleanField %}\n    {% set dummy = field.__setattr__('boolean_mode', true) %}\n  {% endif %}\n  <div class=\"ternary-field {% if field.errors %} error {% endif %}\">\n    <div class=\"ternary-field-label\"><label for=\"{{ field.id }}\">{{ field.label.text | string | forceescape }}</label></div>\n    <div class=\"ternary-field-widget\">{{ field(**kwargs)|safe }}</div>\n    {% if field.errors %}\n      <ul class=errors>\n      {% for error in field.errors %}\n        <li>{{ error }}</li>\n      {% endfor %}\n      </ul>\n    {% endif %}\n  </div>\n{% endmacro %}\n\n\n{% macro render_simple_field(field) %}\n  <span class=\"label {% if field.errors %}error{% endif %}\"><label for=\"{{ field.id }}\">{{ field.label.text | string | forceescape }}</label></span>\n  <span {% if field.errors %} class=\"error\" {% endif %}>{{ field(**kwargs)|safe }}\n  {% if field.errors %}\n    <ul class=errors>\n    {% for error in field.errors %}\n      <li>{{ error }}</li>\n    {% endfor %}\n    </ul>\n  {% endif %}\n  </span>\n{% endmacro %}\n\n\n{% macro render_nolabel_field(field) %}\n    <span>\n    {{ field(**kwargs)|safe }}\n        {% if field.errors %}\n            <span class=\"error\">\n      {% if field.errors %}\n          <ul class=errors>\n        {% for error in field.errors %}\n            <li>{{ error }}</li>\n        {% endfor %}\n        </ul>\n      {% endif %}\n      </span>\n        {% endif %}\n    </span>\n{% endmacro %}\n\n\n{% macro render_button(field) %}\n  {{ field(**kwargs)|safe }}\n{% endmacro %}\n\n{% macro render_fieldlist_with_inline_errors(fieldlist) %}\n  {# Specialized macro for FieldList(FormField(...)) that renders errors inline with each field #}\n  <div {% if fieldlist.errors %} class=\"error\" {% endif %}>{{ _(fieldlist.label.text | string) }}</div>\n  <div {% if fieldlist.errors %} class=\"error\" {% endif %}>\n    <ul id=\"{{ fieldlist.id }}\">\n      {% for entry in fieldlist %}\n        <li {% if entry.errors %} class=\"error\" {% endif %}>\n          <label for=\"{{ entry.id }}\" {% if entry.errors %} class=\"error\" {% endif %}>{{ _(fieldlist.label.text | string) }}-{{ loop.index0 }}</label>\n          <table id=\"{{ entry.id }}\" {% if entry.errors %} class=\"error\" {% endif %}>\n            <tbody>\n              {% for subfield in entry %}\n                <tr {% if subfield.errors %} class=\"error\" {% endif %}>\n                  <th {% if subfield.errors %} class=\"error\" {% endif %}><label for=\"{{ subfield.id }}\" {% if subfield.errors %} class=\"error\" {% endif %}>{{ subfield.label.text | string }}</label></th>\n                  <td {% if subfield.errors %} class=\"error\" {% endif %}>\n                    {{ subfield(**kwargs)|safe }}\n                    {% if subfield.errors %}\n                      <ul class=\"errors\">\n                        {% for error in subfield.errors %}\n                          <li class=\"error\">{{ error }}</li>\n                        {% endfor %}\n                      </ul>\n                    {% endif %}\n                  </td>\n                </tr>\n              {% endfor %}\n            </tbody>\n          </table>\n        </li>\n      {% endfor %}\n    </ul>\n  </div>\n{% endmacro %}\n\n{% macro render_conditions_fieldlist_of_formfields_as_table(fieldlist, table_id=\"rulesTable\") %}\n  <div class=\"fieldlist_formfields\" id=\"{{ table_id }}\">\n    <div class=\"fieldlist-header\">\n      {% for subfield in fieldlist[0] %}\n        <div class=\"fieldlist-header-cell\">{{ subfield.label.text | string }}</div>\n      {% endfor %}\n      <div class=\"fieldlist-header-cell\">{{ _('Actions') }}</div>\n    </div>\n    <div class=\"fieldlist-body\">\n      {% for form_row in fieldlist %}\n        <div class=\"fieldlist-row {% if form_row.errors %}error-row{% endif %}\">\n          {% for subfield in form_row %}\n            <div class=\"fieldlist-cell\">\n\n              {{ subfield()|safe }}\n              {% if subfield.errors %}\n                <ul class=\"errors\">\n                  {% for error in subfield.errors %}\n                    <li class=\"error\">{{ error }}</li>\n                  {% endfor %}\n                </ul>\n              {% endif %}\n            </div>\n          {% endfor %}\n          <div class=\"fieldlist-cell fieldlist-actions\">\n            <button type=\"button\" class=\"addRuleRow\" title=\"{{ _('Add a row/rule after') }}\">+</button>\n            <button type=\"button\" class=\"removeRuleRow\" title=\"{{ _('Remove this row/rule') }}\">-</button>\n            <button type=\"button\" class=\"verifyRuleRow\" title=\"{{ _('Verify this rule against current snapshot') }}\">✓</button>\n          </div>\n        </div>\n      {% endfor %}\n    </div>\n  </div>\n{% endmacro %}\n\n\n{% macro playwright_warning() %}\n    <p><strong>{{ _('Error - This watch needs Chrome (with playwright/sockpuppetbrowser), but Chrome based fetching is not enabled.') }}</strong> {{ _('Alternatively try our') }} <a href=\"https://changedetection.io\">{{ _('very affordable subscription based service which has all this setup for you') }}</a>.</p>\n    <p>{{ _('You may need to') }} <a href=\"https://github.com/dgtlmoon/changedetection.io/blob/09ebc6ec6338545bdd694dc6eee57f2e9d2b8075/docker-compose.yml#L31\">{{ _('Enable playwright environment variable') }}</a> {{ _('and uncomment the') }} <strong>sockpuppetbrowser</strong> {{ _('in the') }} <a href=\"https://github.com/dgtlmoon/changedetection.io/blob/master/docker-compose.yml\">docker-compose.yml</a> {{ _('file') }}.</p>\n    <br>\n{% endmacro %}\n\n{% macro render_time_schedule_form(form, available_timezones, timezone_default_config) %}\n    <style>\n    .day-schedule *, .day-schedule select {\n        display: inline-block;\n    }\n\n    .day-schedule label[for*=\"time_schedule_limit-\"][for$=\"-enabled\"] {\n        min-width: 6rem;\n        font-weight: bold;\n    }\n    .day-schedule label {\n        font-weight: normal;\n    }\n\n    .day-schedule table label {\n        padding-left: 0.5rem;\n        padding-right: 0.5rem;\n    }\n    #timespan-warning, input[id*='time_schedule_limit-timezone'].error {\n        color: #ff0000;\n    }\n    .day-schedule.warning table {\n        background-color: #ffbbc2;\n    }\n    ul#day-wrapper {\n        list-style: none;\n    }\n    #timezone-info > * {\n        display: inline-block;\n    }\n\n    #scheduler-icon-label {\n        background-position: left center;\n        background-repeat: no-repeat;\n        background-size: contain;\n        display: inline-block;\n        vertical-align: middle;\n        padding-left: 50px;\n        background-image: url({{ url_for('static_content', group='images', filename='schedule.svg') }});\n    }\n    #timespan-warning {\n        display: none;\n    }\n    </style>\n    <br>\n\n    {% if timezone_default_config %}\n    <div>\n        <span id=\"scheduler-icon-label\" style=\"\">\n            {{ render_checkbox_field(form.time_schedule_limit.enabled) }}\n            <div class=\"pure-form-message-inline\">\n                {{ _('Set a hourly/week day schedule') }}\n            </div>\n        </span>\n\n    </div>\n    <br>\n    <div id=\"schedule-day-limits-wrapper\">\n        <label>{{ _('Schedule time limits') }}</label><a data-template=\"business-hours\"\n                                              class=\"set-schedule pure-button button-secondary button-xsmall\">{{ _('Business hours') }}</a>\n        <a data-template=\"weekend\" class=\"set-schedule pure-button button-secondary button-xsmall\">{{ _('Weekends') }}</a>\n        <a data-template=\"reset\" class=\"set-schedule pure-button button-xsmall\">{{ _('Reset') }}</a><br>\n        <br>\n\n        <ul id=\"day-wrapper\">\n            {% for day in ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday'] %}\n                <li class=\"day-schedule\" id=\"schedule-{{ day }}\">\n                    {{ render_nolabel_field(form.time_schedule_limit[day]) }}\n                </li>\n            {% endfor %}\n            <li id=\"timespan-warning\">{{ _(\"Warning, one or more of your 'days' has a duration that would extend into the next day.\") }}<br>\n            {{ _('This could have unintended consequences.') }}</li>\n            <li id=\"timezone-info\">\n                {{ render_field(form.time_schedule_limit.timezone, placeholder=timezone_default_config) }} <span id=\"local-time-in-tz\"></span>\n                <datalist id=\"timezones\" style=\"display: none;\">\n                    {%- for timezone in available_timezones -%}<option value=\"{{ timezone }}\">{{ timezone }}</option>{%- endfor -%}\n                </datalist>\n            </li>\n        </ul>\n    <br>\n        <span class=\"pure-form-message-inline\">\n         <a href=\"https://changedetection.io/tutorial/checking-web-pages-changes-according-schedule\">{{ _('More help and examples about using the scheduler') }}</a>\n        </span>\n    </div>\n    {% else %}\n        <span class=\"pure-form-message-inline\">\n            {{ _('Want to use a time schedule?') }} <a href=\"{{url_for('settings.settings_page')}}#timedate\">{{ _('First confirm/save your Time Zone Settings') }}</a>\n        </span>\n        <br>\n    {% endif %}\n\n{% endmacro %}\n\n{% macro highlight_trigger_ignored_explainer() %}\n                <p>\n                    <span title=\"{{ _('Triggers a change if this text appears, AND something changed in the document.') }}\" style=\"background-color: var(--highlight-trigger-text-bg-color); color: #fff; padding: 4px; border-radius: 2px; margin-right: 4px;\">{{ _('Triggered text') }}</span>\n                    <span title=\"{{ _('Ignored for calculating changes, but still shown.') }}\" style=\"background-color: var(--highlight-ignored-text-bg-color); color: #fff; padding: 4px; border-radius: 2px; margin-right: 4px;\">{{ _('Ignored text') }}</span>\n                    <span title=\"{{ _('No change-detection will occur because this text exists.') }}\" style=\"background-color: var(--highlight-blocked-text-bg-color); color: #fff; padding: 4px; border-radius: 2px; margin-right: 4px;\">{{ _('Blocked text') }}</span>\n                </p>\n{% endmacro %}"
  },
  {
    "path": "changedetectionio/templates/base.html",
    "content": "<!DOCTYPE html>\n<html lang=\"{{ get_locale()|replace('_', '-') }}\" data-darkmode=\"{{ get_darkmode_state() }}\">\n\n  <head>\n    <meta charset=\"utf-8\" >\n    <meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\" >\n    <meta name=\"description\" content=\"Self hosted website change detection.\" >\n    <meta name=\"robots\" content=\"noindex\">\n    <title>Change Detection{{extra_title}}\n    {% if app_rss_token %}\n        \n\n        {% if rss_uuid_feed %}\n        \n\n        {%- endif -%}\n    {%- endif -%}\n    \n    \n    \n    {% if extra_stylesheets %}\n      {% for m in extra_stylesheets %}\n        \n      {% endfor %}\n    {% endif %}\n\n    \n    \n    \n    \n    \n    \n    \n    \n    \n    \n    \n    \n    \n    {% if socket_io_enabled %}\n    \n    \n    {% endif %}\n  \n\n  \n    
\n    
\n      
\n\n        {% if has_password and not current_user.is_authenticated %}\n          \n            ChangeDetection.io\n        {% else %}\n          \n            ChangeDetection.io\n        {% endif %}\n        {% if current_diff_url and is_safe_valid_url(current_diff_url) %}\n          \n            {{ current_diff_url }}\n        {% else %}\n          {% if new_version_available and not(has_password and not current_user.is_authenticated) %}\n            \n              A new version is available\n            \n          {% endif %}\n        {% endif %}\n\n        
    \n          \n          {% include \"menu.html\" %}\n\n          {% if current_user.is_authenticated or not has_password %}\n            {% if not current_diff_url %}\n              
  • \n                \n              
    • \n\n            {% endif %}\n          {% endif %}\n\n          
  • \n                \n                  \n                \n          
    • \n          \n          
  • \n            \n          
    • \n        
\n      
\n\n      \n      
\n      
\n        \n      
\n      
\n      
\n\n    
\n    {% if hosted_sticky %}\n      \n    {% endif %}\n    {% if left_sticky %}\n      
\n      
\n    {% endif %}\n    {% if right_sticky %}\n      
{{ right_sticky }}
\n    {% endif %}\n    
\n        
\n            
\n                
Try our Chrome extension
\n                
\n                    \n                        \"Chrome\n                        Chrome Webstore\n                    \n                
\n                Easily add the current web-page from your browser directly into your changedetection.io tool, more great features coming soon!\n                
Changedetection.io needs your support!
\n                
\n                    You can help us by supporting changedetection.io on these platforms;\n                
\n                
\n                
\n                
\n                    The more popular changedetection.io is, the more time we can dedicate to adding amazing features!\n                
\n                
\n                    Many thanks :)
\n                
\n                
\n                    changedetection.io team\n                
\n            
\n        
\n\n        
\n{#\n          {% if current_user.is_authenticated or not has_password %}\n          \n          {% endif %}\n#}\n          
\n            
\n              {% block header %}{% endblock %}\n            
\n\n            {% with messages = get_flashed_messages(with_categories = true) %}\n            {% if messages %}\n              
    \n                {% for category, message in messages %}\n                  
  • {{ message }}
    • \n                {% endfor %}\n              
\n            {% endif %}\n            {% endwith %}\n            {% if session['share-link'] %}\n              
    \n                
  • \n                  Share this link:\n                  {{ session['share-link'] }}\n                  \n                
    • \n              
\n            {% endif %}\n            {% block content %}{% endblock %}\n          
\n        
\n    
\n    \n    \n\n    
 {{ _('Checking now') }}
\n    
{{ _('Real-time updates offline') }}
\n    {% if bottom_horizontal_offscreen_contents %}\n        
\n            {{ bottom_horizontal_offscreen_contents|safe }}\n        
\n    {% endif %}\n\n    \n    \n      
\n        
{{ _('Select Language') }}
\n      
\n      
\n        
\n          {% for locale, lang_data in available_languages.items()|sort %}\n          \n             {{ lang_data.name }}\n          \n          {% endfor %}\n        
\n        \n        
\n            {{ _('Language support is in beta, please help us improve by opening a PR on GitHub with any updates.') }}\n        
\n      
\n      
\n        \n      
\n    \n\n    \n    {% if current_user.is_authenticated or not has_password %}\n    \n      
\n        
{{ _('Search') }}
\n      
\n      
\n        
\n          
\n            \n            \n            \n          
\n        
\n      
\n      
\n        \n        \n      
\n    \n    {% endif %}\n\n\n  \n\n\n    \n    \n    \n    \n  \n\n\n"
  },
  {
    "path": "changedetectionio/templates/edit/include_subtract.html",
    "content": "                    
\n                        {% set field = render_field(form.include_filters,\n                            rows=5,\n                            placeholder=has_tag_filters_extra+\"#example\nxpath://body/div/span[contains(@class, 'example-class')]\",\n                            class=\"m-d\")\n                        %}\n                        {{ field }}\n                        {% if '/text()' in  field %}\n                          Note!: //text() function does not work where the <element> contains <![CDATA[]]>
\n                        {% endif %}\n                        One CSS, xPath 1 & 2, JSON Path/JQ selector per line, any rules that matches will be used.
\n                        Show advanced help and tips
\n                    
    \n                        
  • CSS - Limit text to this CSS rule, only text matching this CSS rule is included.
    • \n                        
  • JSON - Limit text to this JSON rule, using either JSONPath or jq (if installed).\n                            
      \n                                
    • JSONPath: Prefix with json:, use json:$ to force re-formatting if required,  test your JSONPath here.
      • \n                                {% if jq_support %}\n                                
    • jq: Prefix with jq: and test your jq here. Using jq allows for complex filtering and processing of JSON data with built-in functions, regex, filtering, and more. See examples and documentation here. Prefix jqraw: outputs the results as text instead of a JSON list.
      • \n                                {% else %}\n                                
    • jq support not installed
      • \n                                {% endif %}\n                            
    \n                        
    • \n                        
  • XPath - Limit text to this XPath rule, simply start with a forward-slash. To specify XPath to be used explicitly or the XPath rule starts with an XPath function: Prefix with xpath:\n                            
      \n                                
    • Example:  //*[contains(@class, 'sametext')] or xpath:count(//*[contains(@class, 'sametext')]), test your XPath here
      • \n                                
    • Example: Get all titles from an RSS feed //title/text()
      • \n                                
    • To use XPath1.0: Prefix with xpath1:
      • \n                            
    \n                            
    • \n                    
  • \n                        Please be sure that you thoroughly understand how to write CSS, JSONPath, XPath{% if jq_support %}, or jq selector{%endif%} rules before filing an issue on GitHub! here for more CSS selector help.
    \n                    
    • \n                    
\n\n                \n                    
\n                
\n                    {{ render_field(form.subtractive_selectors, rows=5, placeholder=has_tag_filters_extra+\"header\nfooter\nnav\n.stockticker\n//*[contains(text(), 'Advertisement')]\") }}\n                    \n                        
    \n                          
  •  Remove HTML element(s) by CSS and XPath selectors before text conversion. 
    • \n                          
  •  Don't paste HTML here, use only CSS and XPath selectors 
    • \n                          
  •  Add multiple elements, CSS or XPath selectors per line to ignore multiple parts of the HTML. 
    • \n                        
\n                      \n                
\n"
  },
  {
    "path": "changedetectionio/templates/edit/text-options.html",
    "content": "\n                
\n                    
\n                        {{ render_field(form.trigger_text, rows=5, placeholder=\"Some text to wait for in a line\n/some.regex\\d{2}/ for case-INsensitive regex\n\") }}\n                        \n                    
    \n                        
  • {{ _('Text to wait for before triggering a change/notification, all text and regex are tested case-insensitive.') }}
    • \n                        
  • {{ _('Trigger text is processed from the result-text that comes out of any CSS/JSON Filters for this monitor') }}
    • \n                        
  • {{ _('Each line is processed separately (think of each line as \"OR\")') }}
    • \n                        
  • {{ _('Note: Wrap in forward slash / to use regex example:') }} /foo\\d/
    • \n                        
  • {{ _('You can also use')}} {{ _('conditions')}} - {{ _('\"Page text\" - with Contains, Starts With, Not Contains and many more' ) }} /foo\\d/
    • \n                    
\n                        \n                    
\n                
\n                
\n                    {{ render_field(form.ignore_text, rows=5, placeholder=\"Some text to ignore in a line\n/some.regex\\d{2}/ for case-INsensitive regex\n\") }}\n                    \n                        
    \n                            
  • {{ _('Matching text will be ignored in the text snapshot (you can still see it but it wont trigger a change)') }}
    • \n                            
  • {{ _('Each line processed separately, any line matching will be ignored (removed before creating the checksum)') }}
    • \n                            
  • {{ _('Regular Expression support, wrap the entire line in forward slash') }} /regex/
    • \n                            
  • {{ _('Changing this will affect the comparison checksum which may trigger an alert') }}
    • \n                        
\n                \n                

\n                    
\n                      {{ render_ternary_field(form.strip_ignored_lines) }}\n                    
\n                
\n\n                
\n                    
\n                        {{ render_field(form.text_should_not_be_present, rows=5, placeholder=\"For example: Out of stock\nSold out\nNot in stock\nUnavailable\") }}\n                        \n                            
    \n                                
  • {{ _('Block change-detection while this text is on the page, all text and regex are tested case-insensitive, good for waiting for when a product is available again') }}
    • \n                                
  • {{ _('Block text is processed from the result-text that comes out of any CSS/JSON Filters for this monitor') }}
    • \n                                
  • {{ _('All lines here must not exist (think of each line as \"OR\")') }}
    • \n                                
  • {{ _('Note: Wrap in forward slash / to use regex example:') }} /foo\\d/
    • \n                            
\n                        \n                    
\n                
\n                
\n                    
\n                        {{ render_field(form.extract_text, rows=5, placeholder=\"/.+?\\d+ comments.+?/\n or\nkeyword\") }}\n                        \n                    
    \n                        
  • {{ _('Extracts text in the final output (line by line) after other filters using regular expressions or string match:') }}\n                            
      \n                                
    • {{ _('Regular expression - example') }} /reports.+?2022/i
      • \n                                
    • {{ _('Don\\'t forget to consider the white-space at the start of a line') }} /.+?reports.+?2022/i
      • \n                                
    • {{ _('Use') }} //(?aiLmsux)) {{ _('type flags (more') }} {{ _('information here') }})
      • \n                                
    • {{ _('Keyword example - example') }} Out of stock
      • \n                                
    • {{ _('Use groups to extract just that text - example') }} /reports.+?(\\d+)/i {{ _('returns a list of years only') }}
      • \n                                
    • {{ _('Example - match lines containing a keyword') }} /.*icecream.*/
      • \n                            
    \n                        
    • \n                        
  • {{ _('One line per regular-expression/string match') }}
    • \n                    
\n                        \n                    
\n                
\n"
  },
  {
    "path": "changedetectionio/templates/login.html",
    "content": "{% extends 'base.html' %}\n\n{% block content %}\n
\n 
\n    
\n        \n        \n        
\n            
\n                \n                \n                \n            
\n            
\n                \n            
\n        
\n    
\n  
\n 
\n\n{% endblock %}\n"
  },
  {
    "path": "changedetectionio/templates/menu.html",
    "content": "{# Menu items template - used for both desktop and mobile menus #}\n{# CSS media queries handle which version displays - no need for conditional classes #}\n\n\n{% if current_user.is_authenticated or not has_password %}\n{% if not current_diff_url %}\n    
  • \n        {{ _('GROUPS') }}\n    
    • \n    
  • \n        {{ _('SETTINGS') }}\n    
    • \n    
  • \n        {{ _('IMPORT') }}\n    
    • \n    
  • \n        \"{%\n    
    • \n    
  • \n        \"{%\n    
    • \n{% else %}\n    
  • \n        {{ _('EDIT') }}\n    
    • \n{% endif %}\n{%- if current_user.is_authenticated -%}\n    
  • \n        {{ _('LOG OUT') }}\n    
    • \n{%- endif -%}\n\n{% else %}\n    
  • \n        {{ _('Website Change Detection and Notification.') }}\n    
    • \n{% endif %}\n    
  • \n        \n        \n        \n            {% include \"svgs/github.svg\" %}\n        \n    
    • \n\n"
  },
  {
    "path": "changedetectionio/test_cli_opts.sh",
    "content": "#!/bin/bash\n# Test script for CLI options - Parallel execution\n# Tests -u, -uN, -r, -b flags\n\nset -u  # Exit on undefined variables\n\n# Color output\nRED='\\033[0;31m'\nGREEN='\\033[0;32m'\nYELLOW='\\033[1;33m'\nNC='\\033[0m' # No Color\n\n# Test results directory (for parallel safety)\nTEST_RESULTS_DIR=\"/tmp/cli-test-results-$$\"\nmkdir -p \"$TEST_RESULTS_DIR\"\n\n# Cleanup function\ncleanup() {\n    echo \"\"\n    echo \"=== Cleaning up test directories ===\"\n    rm -rf /tmp/cli-test-* 2>/dev/null || true\n    rm -rf \"$TEST_RESULTS_DIR\" 2>/dev/null || true\n    # Kill any hanging processes\n    pkill -f \"changedetection.py.*cli-test\" 2>/dev/null || true\n}\ntrap cleanup EXIT\n\n# Helper to record test result\nrecord_result() {\n    local test_num=$1\n    local status=$2  # pass or fail\n    local message=$3\n\n    echo \"$status|$message\" > \"$TEST_RESULTS_DIR/test_${test_num}.result\"\n}\n\n# Run a test in background\nrun_test() {\n    local test_num=$1\n    local test_name=$2\n    local test_func=$3\n\n    (\n        echo -e \"${YELLOW}[Test $test_num]${NC} $test_name\"\n        if $test_func \"$test_num\"; then\n            record_result \"$test_num\" \"pass\" \"$test_name\"\n            echo -e \"${GREEN}✓ PASS${NC}: $test_name\"\n        else\n            record_result \"$test_num\" \"fail\" \"$test_name\"\n            echo -e \"${RED}✗ FAIL${NC}: $test_name\"\n        fi\n    ) &\n}\n\n# =============================================================================\n# Test Functions (each runs independently)\n# =============================================================================\n\ntest_help_flag() {\n    local test_id=$1\n    timeout 3 python3 changedetection.py --help 2>&1 | grep -q \"Add URLs on startup\"\n}\n\ntest_version_flag() {\n    local test_id=$1\n    timeout 3 python3 changedetection.py --version 2>&1 | grep -qE \"changedetection.io [0-9]+\\.[0-9]+\"\n}\n\ntest_single_url() {\n    local test_id=$1\n    local dir=\"/tmp/cli-test-single-${test_id}-$$\"\n    timeout 10 python3 changedetection.py -d \"$dir\" -C -u https://example.com -b &>/dev/null\n    # Count watch directories (UUID directories containing watch.json)\n    [ \"$(find \"$dir\" -mindepth 2 -maxdepth 2 -name 'watch.json' | wc -l)\" -eq 1 ]\n}\n\ntest_multiple_urls() {\n    local test_id=$1\n    local dir=\"/tmp/cli-test-multi-${test_id}-$$\"\n    timeout 12 python3 changedetection.py -d \"$dir\" -C \\\n        -u https://example.com \\\n        -u https://github.com \\\n        -u https://httpbin.org \\\n        -b &>/dev/null\n    # Count watch directories (UUID directories containing watch.json)\n    [ \"$(find \"$dir\" -mindepth 2 -maxdepth 2 -name 'watch.json' | wc -l)\" -eq 3 ]\n}\n\ntest_url_with_options() {\n    local test_id=$1\n    local dir=\"/tmp/cli-test-opts-${test_id}-$$\"\n    timeout 10 python3 changedetection.py -d \"$dir\" -C \\\n        -u https://example.com \\\n        -u0 '{\"title\":\"Test Site\",\"processor\":\"text_json_diff\"}' \\\n        -b &>/dev/null\n    # Check that at least one watch.json contains the title \"Test Site\"\n    python3 -c \"\nimport json, glob, sys\nwatch_files = glob.glob('$dir/*/watch.json')\nfor wf in watch_files:\n    with open(wf) as f:\n        data = json.load(f)\n        if data.get('title') == 'Test Site':\n            sys.exit(0)\nsys.exit(1)\n\"\n}\n\ntest_multiple_urls_with_options() {\n    local test_id=$1\n    local dir=\"/tmp/cli-test-multi-opts-${test_id}-$$\"\n    timeout 12 python3 changedetection.py -d \"$dir\" -C \\\n        -u https://example.com \\\n        -u0 '{\"title\":\"Site One\"}' \\\n        -u https://github.com \\\n        -u1 '{\"title\":\"Site Two\"}' \\\n        -b &>/dev/null\n    # Check that we have 2 watches and both titles are present\n    python3 -c \"\nimport json, glob, sys\nwatch_files = glob.glob('$dir/*/watch.json')\nif len(watch_files) != 2:\n    sys.exit(1)\ntitles = []\nfor wf in watch_files:\n    with open(wf) as f:\n        data = json.load(f)\n        titles.append(data.get('title'))\nsys.exit(0 if 'Site One' in titles and 'Site Two' in titles else 1)\n\"\n}\n\ntest_batch_mode_exit() {\n    local test_id=$1\n    local dir=\"/tmp/cli-test-batch-${test_id}-$$\"\n    local start=$(date +%s)\n    timeout 15 python3 changedetection.py -d \"$dir\" -C \\\n        -u https://example.com \\\n        -b &>/dev/null\n    local end=$(date +%s)\n    local elapsed=$((end - start))\n    [ $elapsed -lt 14 ]\n}\n\ntest_recheck_all() {\n    local test_id=$1\n    local dir=\"/tmp/cli-test-recheck-all-${test_id}-$$\"\n    # Create a watch using CLI, then recheck it\n    timeout 10 python3 changedetection.py -d \"$dir\" -C -u https://example.com -b &>/dev/null\n    # Now recheck all watches\n    timeout 10 python3 changedetection.py -d \"$dir\" -r all -b 2>&1 | grep -q \"Queuing\"\n}\n\ntest_recheck_specific() {\n    local test_id=$1\n    local dir=\"/tmp/cli-test-recheck-uuid-${test_id}-$$\"\n    # Create 2 watches using CLI\n    timeout 12 python3 changedetection.py -d \"$dir\" -C \\\n        -u https://example.com \\\n        -u https://github.com \\\n        -b &>/dev/null\n    # Get the UUIDs that were created\n    local uuids=$(find \"$dir\" -mindepth 2 -maxdepth 2 -name 'watch.json' -exec dirname {} \\; | xargs -n1 basename | tr '\\n' ',' | sed 's/,$//')\n    # Now recheck specific UUIDs\n    timeout 10 python3 changedetection.py -d \"$dir\" -r \"$uuids\" -b 2>&1 | grep -q \"Queuing\"\n}\n\ntest_combined_operations() {\n    local test_id=$1\n    local dir=\"/tmp/cli-test-combined-${test_id}-$$\"\n    timeout 12 python3 changedetection.py -d \"$dir\" -C \\\n        -u https://example.com \\\n        -u https://github.com \\\n        -r all \\\n        -b &>/dev/null\n    # Count watch directories (UUID directories containing watch.json)\n    [ \"$(find \"$dir\" -mindepth 2 -maxdepth 2 -name 'watch.json' | wc -l)\" -eq 2 ]\n}\n\ntest_invalid_json() {\n    local test_id=$1\n    local dir=\"/tmp/cli-test-invalid-${test_id}-$$\"\n    timeout 5 python3 changedetection.py -d \"$dir\" -C \\\n        -u https://example.com \\\n        -u0 'invalid json here' \\\n        2>&1 | grep -qi \"invalid json\\|json decode error\"\n}\n\ntest_create_directory() {\n    local test_id=$1\n    local dir=\"/tmp/cli-test-create-${test_id}-$$/nested/path\"\n    timeout 10 python3 changedetection.py -d \"$dir\" -C \\\n        -u https://example.com \\\n        -b &>/dev/null\n    [ -d \"$dir\" ]\n}\n\n# =============================================================================\n# Main Test Execution\n# =============================================================================\n\necho \"==========================================\"\necho \"  CLI Options Test Suite (Parallel)\"\necho \"==========================================\"\necho \"\"\n\n# Launch all tests in parallel\nrun_test 1 \"Help flag (--help) shows usage without initialization\" test_help_flag\nrun_test 2 \"Version flag (--version) displays version\" test_version_flag\nrun_test 3 \"Add single URL with -u flag\" test_single_url\nrun_test 4 \"Add multiple URLs with multiple -u flags\" test_multiple_urls\nrun_test 5 \"Add URL with JSON options using -u0\" test_url_with_options\nrun_test 6 \"Add multiple URLs with different options (-u0, -u1)\" test_multiple_urls_with_options\nrun_test 7 \"Batch mode (-b) exits automatically after processing\" test_batch_mode_exit\nrun_test 8 \"Recheck all watches with -r all\" test_recheck_all\nrun_test 9 \"Recheck specific watches with -r UUID\" test_recheck_specific\nrun_test 10 \"Combined: Add URLs and recheck all with -u and -r all\" test_combined_operations\nrun_test 11 \"Invalid JSON in -u0 option should show error\" test_invalid_json\nrun_test 12 \"Create datastore directory with -C flag\" test_create_directory\n\n# Wait for all tests to complete\necho \"\"\necho \"Waiting for all tests to complete...\"\nwait\n\n# Collect results\necho \"\"\necho \"==========================================\"\necho \"  Test Summary\"\necho \"==========================================\"\n\nTESTS_RUN=0\nTESTS_PASSED=0\nTESTS_FAILED=0\n\nfor result_file in \"$TEST_RESULTS_DIR\"/test_*.result; do\n    if [ -f \"$result_file\" ]; then\n        TESTS_RUN=$((TESTS_RUN + 1))\n        status=$(cut -d'|' -f1 < \"$result_file\")\n        if [ \"$status\" = \"pass\" ]; then\n            TESTS_PASSED=$((TESTS_PASSED + 1))\n        else\n            TESTS_FAILED=$((TESTS_FAILED + 1))\n        fi\n    fi\ndone\n\necho \"Tests run:    $TESTS_RUN\"\necho -e \"${GREEN}Tests passed: $TESTS_PASSED${NC}\"\nif [ $TESTS_FAILED -gt 0 ]; then\n    echo -e \"${RED}Tests failed: $TESTS_FAILED${NC}\"\nelse\n    echo -e \"${GREEN}Tests failed: $TESTS_FAILED${NC}\"\nfi\necho \"==========================================\"\necho \"\"\n\n# Exit with appropriate code\nif [ $TESTS_FAILED -gt 0 ]; then\n    echo -e \"${RED}Some tests failed!${NC}\"\n    exit 1\nelse\n    echo -e \"${GREEN}All tests passed!${NC}\"\n    exit 0\nfi\n"
  },
  {
    "path": "changedetectionio/tests/__init__.py",
    "content": "\"\"\"Tests for the app.\"\"\"\n\n"
  },
  {
    "path": "changedetectionio/tests/apprise/test_apprise_asset.py",
    "content": "import pytest\nfrom apprise import AppriseAsset\n\nfrom changedetectionio.apprise_asset import (\n    APPRISE_APP_DESC,\n    APPRISE_APP_ID,\n    APPRISE_APP_URL,\n    APPRISE_AVATAR_URL,\n)\n\n\n@pytest.fixture(scope=\"function\")\ndef apprise_asset() -> AppriseAsset:\n    from changedetectionio.apprise_asset import apprise_asset\n\n    return apprise_asset\n\n\ndef test_apprise_asset_init(apprise_asset: AppriseAsset):\n    assert isinstance(apprise_asset, AppriseAsset)\n    assert apprise_asset.app_id == APPRISE_APP_ID\n    assert apprise_asset.app_desc == APPRISE_APP_DESC\n    assert apprise_asset.app_url == APPRISE_APP_URL\n    assert apprise_asset.image_url_logo == APPRISE_AVATAR_URL\n"
  },
  {
    "path": "changedetectionio/tests/apprise/test_apprise_custom_api_call.py",
    "content": "import json\nfrom unittest.mock import patch\n\nimport pytest\nimport requests\nfrom apprise.utils.parse import parse_url as apprise_parse_url\n\nfrom ...apprise_plugin.custom_handlers import (\n    _get_auth,\n    _get_headers,\n    _get_params,\n    apprise_http_custom_handler,\n    SUPPORTED_HTTP_METHODS,\n)\n\n\n@pytest.mark.parametrize(\n    \"url,expected_auth\",\n    [\n        (\"get://user:pass@localhost:9999\", (\"user\", \"pass\")),\n        (\"get://user@localhost:9999\", \"user\"),\n        (\"get://localhost:9999\", \"\"),\n        (\"get://user%20name:pass%20word@localhost:9999\", (\"user name\", \"pass word\")),\n    ],\n)\ndef test_get_auth(url, expected_auth):\n    \"\"\"Test authentication extraction with various URL formats.\"\"\"\n    parsed_url = apprise_parse_url(url)\n    assert _get_auth(parsed_url) == expected_auth\n\n\n@pytest.mark.parametrize(\n    \"url,body,expected_content_type\",\n    [\n        (\n            \"get://localhost:9999?+content-type=application/xml\",\n            \"test\",\n            \"application/xml\",\n        ),\n        (\"get://localhost:9999\", '{\"key\": \"value\"}', \"application/json; charset=utf-8\"),\n        (\"get://localhost:9999\", \"plain text\", None),\n        (\"get://localhost:9999?+content-type=text/plain\", \"test\", \"text/plain\"),\n    ],\n)\ndef test_get_headers(url, body, expected_content_type):\n    \"\"\"Test header extraction and content type detection.\"\"\"\n    parsed_url = apprise_parse_url(url)\n    headers = _get_headers(parsed_url, body)\n\n    if expected_content_type:\n        assert headers.get(\"Content-Type\") == expected_content_type\n\n\n@pytest.mark.parametrize(\n    \"url,expected_params\",\n    [\n        (\"get://localhost:9999?param1=value1\", {\"param1\": \"value1\"}),\n        (\"get://localhost:9999?param1=value1&-param2=ignored\", {\"param1\": \"value1\"}),\n        (\"get://localhost:9999?param1=value1&+header=test\", {\"param1\": \"value1\"}),\n        (\n            \"get://localhost:9999?encoded%20param=encoded%20value\",\n            {\"encoded param\": \"encoded value\"},\n        ),\n    ],\n)\ndef test_get_params(url, expected_params):\n    \"\"\"Test parameter extraction with URL encoding and exclusion logic.\"\"\"\n    parsed_url = apprise_parse_url(url)\n    params = _get_params(parsed_url)\n    assert dict(params) == expected_params\n\n\n@pytest.mark.parametrize(\n    \"url,schema,method\",\n    [\n        (\"get://localhost:9999\", \"get\", \"GET\"),\n        (\"post://localhost:9999\", \"post\", \"POST\"),\n        (\"delete://localhost:9999\", \"delete\", \"DELETE\"),\n    ],\n)\n@patch(\"requests.request\")\ndef test_apprise_custom_api_call_success(mock_request, url, schema, method):\n    \"\"\"Test successful API calls with different HTTP methods and schemas.\"\"\"\n    mock_request.return_value.raise_for_status.return_value = None\n\n    meta = {\"url\": url, \"schema\": schema}\n    result = apprise_http_custom_handler(\n        body=\"test body\", title=\"Test Title\", notify_type=\"info\", meta=meta\n    )\n\n    assert result is True\n    mock_request.assert_called_once()\n\n    call_args = mock_request.call_args\n    assert call_args[1][\"method\"] == method.upper()\n    assert call_args[1][\"url\"].startswith(\"http\")\n\n\n@patch(\"requests.request\")\ndef test_apprise_custom_api_call_with_auth(mock_request):\n    \"\"\"Test API call with authentication.\"\"\"\n    mock_request.return_value.raise_for_status.return_value = None\n\n    url = \"get://user:pass@localhost:9999/secure\"\n    meta = {\"url\": url, \"schema\": \"get\"}\n\n    result = apprise_http_custom_handler(\n        body=json.dumps({\"key\": \"value\"}),\n        title=\"Secure Test\",\n        notify_type=\"info\",\n        meta=meta,\n    )\n\n    assert result is True\n    mock_request.assert_called_once()\n    call_args = mock_request.call_args\n    assert call_args[1][\"auth\"] == (\"user\", \"pass\")\n\n\n@pytest.mark.parametrize(\n    \"exception_type,expected_result\",\n    [\n        (requests.RequestException, False),\n        (requests.HTTPError, False),\n        (Exception, False),\n    ],\n)\n@patch(\"requests.request\")\ndef test_apprise_custom_api_call_failure(mock_request, exception_type, expected_result):\n    \"\"\"Test various failure scenarios.\"\"\"\n    url = \"get://localhost:9999/error\"\n    meta = {\"url\": url, \"schema\": \"get\"}\n\n    # Simulate different types of exceptions\n    mock_request.side_effect = exception_type(\"Error occurred\")\n\n    result = apprise_http_custom_handler(\n        body=\"error body\", title=\"Error Test\", notify_type=\"error\", meta=meta\n    )\n\n    assert result == expected_result\n\n\ndef test_invalid_url_parsing():\n    \"\"\"Test handling of invalid URL parsing.\"\"\"\n    meta = {\"url\": \"invalid://url\", \"schema\": \"invalid\"}\n    result = apprise_http_custom_handler(\n        body=\"test\", title=\"Invalid URL\", notify_type=\"info\", meta=meta\n    )\n\n    assert result is False\n\n\n@pytest.mark.parametrize(\n    \"schema,expected_method\",\n    [\n        (http_method, http_method.upper())\n        for http_method in SUPPORTED_HTTP_METHODS\n    ],\n)\n@patch(\"requests.request\")\ndef test_http_methods(mock_request, schema, expected_method):\n    \"\"\"Test all supported HTTP methods.\"\"\"\n    mock_request.return_value.raise_for_status.return_value = None\n\n    url = f\"{schema}://localhost:9999\"\n\n    result = apprise_http_custom_handler(\n        body=\"test body\",\n        title=\"Test Title\",\n        notify_type=\"info\",\n        meta={\"url\": url, \"schema\": schema},\n    )\n\n    assert result is True\n    mock_request.assert_called_once()\n\n    call_args = mock_request.call_args\n    assert call_args[1][\"method\"] == expected_method\n\n\n@pytest.mark.parametrize(\n    \"input_schema,expected_method\",\n    [\n        (f\"{http_method}s\", http_method.upper())\n        for http_method in SUPPORTED_HTTP_METHODS\n    ],\n)\n@patch(\"requests.request\")\ndef test_https_method_conversion(\n    mock_request, input_schema, expected_method\n):\n    \"\"\"Validate that methods ending with 's' use HTTPS and correct HTTP method.\"\"\"\n    mock_request.return_value.raise_for_status.return_value = None\n\n    url = f\"{input_schema}://localhost:9999\"\n\n    result = apprise_http_custom_handler(\n        body=\"test body\",\n        title=\"Test Title\",\n        notify_type=\"info\",\n        meta={\"url\": url, \"schema\": input_schema},\n    )\n\n    assert result is True\n    mock_request.assert_called_once()\n\n    call_args = mock_request.call_args\n\n    assert call_args[1][\"method\"] == expected_method\n    assert call_args[1][\"url\"].startswith(\"https\")\n"
  },
  {
    "path": "changedetectionio/tests/conftest.py",
    "content": "#!/usr/bin/env python3\nimport psutil\nimport time\nfrom threading import Thread\nimport multiprocessing\n\nimport pytest\nimport arrow\nfrom changedetectionio import store\nimport os\nimport sys\n\n# CRITICAL: Set short timeout for tests to prevent 45-second hangs\n# When test server is slow/unresponsive, workers fail fast instead of holding UUIDs for 45s\n# This prevents exponential priority growth from repeated deferrals (priority × 10 each defer)\nos.environ['DEFAULT_SETTINGS_REQUESTS_TIMEOUT'] = '5'\n# Test server runs on localhost (127.0.0.1) which is a private IP.\n# Allow it globally so all existing tests keep working; test_ssrf_protection\n# uses monkeypatch to temporarily override this for its own assertions.\nos.environ['ALLOW_IANA_RESTRICTED_ADDRESSES'] = 'true'\n\nfrom changedetectionio.flask_app import init_app_secret, changedetection_app\nfrom changedetectionio.tests.util import live_server_setup, new_live_server_setup\n\n# https://github.com/pallets/flask/blob/1.1.2/examples/tutorial/tests/test_auth.py\n# Much better boilerplate than the docs\n# https://www.python-boilerplate.com/py3+flask+pytest/\n\nglobal app\n\n# https://loguru.readthedocs.io/en/latest/resources/migration.html#replacing-caplog-fixture-from-pytest-library\n# Show loguru logs only if CICD pytest fails.\nfrom loguru import logger\n@pytest.fixture\ndef reportlog(pytestconfig):\n    logging_plugin = pytestconfig.pluginmanager.getplugin(\"logging-plugin\")\n    handler_id = logger.add(logging_plugin.report_handler, format=\"{message}\")\n    yield\n    logger.remove(handler_id)\n\n\n@pytest.fixture(autouse=True)\ndef per_test_log_file(request):\n    \"\"\"Create a separate log file for each test function with pytest output.\"\"\"\n    import re\n\n    # Create logs directory if it doesn't exist\n    log_dir = os.path.join(os.path.dirname(__file__), \"logs\")\n    os.makedirs(log_dir, exist_ok=True)\n\n    # Generate log filename from test name and worker ID (for parallel runs)\n    test_name = request.node.name\n\n    # Sanitize test name - replace unsafe characters with underscores\n    # Keep only alphanumeric, dash, underscore, and period\n    safe_test_name = re.sub(r'[^\\w\\-.]', '_', test_name)\n\n    # Limit length to avoid filesystem issues (max 200 chars)\n    if len(safe_test_name) > 200:\n        # Keep first 150 chars + hash of full name + last 30 chars\n        import hashlib\n        name_hash = hashlib.md5(test_name.encode()).hexdigest()[:8]\n        safe_test_name = f\"{safe_test_name[:150]}_{name_hash}_{safe_test_name[-30:]}\"\n\n    worker_id = os.environ.get('PYTEST_XDIST_WORKER', 'master')\n    log_file = os.path.join(log_dir, f\"{safe_test_name}_{worker_id}.log\")\n\n    # Add file handler for this test with TRACE level\n    handler_id = logger.add(\n        log_file,\n        format=\"{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {process} | {name}:{function}:{line} - {message}\",\n        level=\"TRACE\",\n        mode=\"w\",  # Overwrite if exists\n        enqueue=True  # Thread-safe\n    )\n\n    logger.info(f\"=== Starting test: {test_name} (worker: {worker_id}) ===\")\n    logger.info(f\"Test location: {request.node.nodeid}\")\n\n    yield\n\n    # Capture test outcome (PASSED/FAILED/SKIPPED/ERROR)\n    outcome = \"UNKNOWN\"\n    exc_info = None\n    stdout = None\n    stderr = None\n\n    if hasattr(request.node, 'rep_call'):\n        outcome = request.node.rep_call.outcome.upper()\n        if request.node.rep_call.failed:\n            exc_info = request.node.rep_call.longreprtext\n        # Capture stdout/stderr from call phase\n        if hasattr(request.node.rep_call, 'sections'):\n            for section_name, section_content in request.node.rep_call.sections:\n                if 'stdout' in section_name.lower():\n                    stdout = section_content\n                elif 'stderr' in section_name.lower():\n                    stderr = section_content\n    elif hasattr(request.node, 'rep_setup'):\n        if request.node.rep_setup.failed:\n            outcome = \"SETUP_FAILED\"\n            exc_info = request.node.rep_setup.longreprtext\n\n    logger.info(f\"=== Test Result: {outcome} ===\")\n\n    if exc_info:\n        logger.error(f\"=== Test Failure Details ===\\n{exc_info}\")\n\n    if stdout:\n        logger.info(f\"=== Captured stdout ===\\n{stdout}\")\n\n    if stderr:\n        logger.warning(f\"=== Captured stderr ===\\n{stderr}\")\n\n    logger.info(f\"=== Finished test: {test_name} ===\")\n    logger.remove(handler_id)\n\n\n@pytest.hookimpl(tryfirst=True, hookwrapper=True)\ndef pytest_runtest_makereport(item, call):\n    \"\"\"Hook to capture test results and attach to the test node.\"\"\"\n    outcome = yield\n    rep = outcome.get_result()\n\n    # Store report on the test node for access in fixtures\n    setattr(item, f\"rep_{rep.when}\", rep)\n\n\n@pytest.fixture\ndef environment(mocker):\n    \"\"\"Mock arrow.now() to return a fixed datetime for testing jinja2 time extension.\"\"\"\n    # Fixed datetime: Wed, 09 Dec 2015 23:33:01 UTC\n    # This is calculated to match the test expectations when offsets are applied\n    fixed_datetime = arrow.Arrow(2015, 12, 9, 23, 33, 1, tzinfo='UTC')\n    # Patch arrow.now in the TimeExtension module where it's actually used\n    mocker.patch('changedetectionio.jinja2_custom.extensions.TimeExtension.arrow.now', return_value=fixed_datetime)\n    return fixed_datetime\n\n\ndef format_memory_human(bytes_value):\n    \"\"\"Format memory in human-readable units (KB, MB, GB)\"\"\"\n    if bytes_value < 1024:\n        return f\"{bytes_value} B\"\n    elif bytes_value < 1024 ** 2:\n        return f\"{bytes_value / 1024:.2f} KB\"\n    elif bytes_value < 1024 ** 3:\n        return f\"{bytes_value / (1024 ** 2):.2f} MB\"\n    else:\n        return f\"{bytes_value / (1024 ** 3):.2f} GB\"\n\ndef track_memory(memory_usage, ):\n    process = psutil.Process(os.getpid())\n    while not memory_usage[\"stop\"]:\n        current_rss = process.memory_info().rss\n        memory_usage[\"peak\"] = max(memory_usage[\"peak\"], current_rss)\n        memory_usage[\"current\"] = current_rss  # Keep updating current\n        time.sleep(0.01)  # Adjust the sleep time as needed\n\n@pytest.fixture(scope='function')\ndef measure_memory_usage(request):\n    memory_usage = {\"peak\": 0, \"current\": 0, \"stop\": False}\n    tracker_thread = Thread(target=track_memory, args=(memory_usage,))\n    tracker_thread.start()\n\n    yield\n\n    memory_usage[\"stop\"] = True\n    tracker_thread.join()\n\n    # Note: psutil returns RSS memory in bytes\n    peak_human = format_memory_human(memory_usage[\"peak\"])\n\n    s = f\"{time.time()} {request.node.fspath} - '{request.node.name}' - Peak memory: {peak_human}\"\n    logger.debug(s)\n\n    with open(\"test-memory.log\", 'a') as f:\n        f.write(f\"{s}\\n\")\n\n    # Assert that the memory usage is less than 200MB\n#    assert peak_memory_kb < 150 * 1024, f\"Memory usage exceeded 150MB: {peak_human}\"\n\n\ndef cleanup(datastore_path):\n    import glob\n    # Unlink test output files\n    for g in [\"*.txt\", \"*.json\", \"*.pdf\"]:\n        files = glob.glob(os.path.join(datastore_path, g))\n        for f in files:\n            if 'proxies.json' in f:\n                # Usually mounted by docker container during test time\n                continue\n            if os.path.isfile(f):\n                os.unlink(f)\n\ndef pytest_configure(config):\n    \"\"\"Configure pytest environment before tests run.\n\n    CRITICAL: Set multiprocessing start method to 'fork' for Python 3.14+ compatibility.\n\n    Python 3.14 changed the default start method from 'fork' to 'forkserver' on Linux.\n    The forkserver method requires all objects to be picklable, but pytest-flask's\n    LiveServer uses nested functions that can't be pickled.\n\n    Setting 'fork' explicitly:\n    - Maintains compatibility with Python 3.10-3.13 (where 'fork' was already default)\n    - Fixes Python 3.14 pickling errors\n    - Only affects Unix-like systems (Windows uses 'spawn' regardless)\n\n    See: https://github.com/python/cpython/issues/126831\n    See: https://docs.python.org/3/whatsnew/3.14.html\n    \"\"\"\n    # Only set if not already set (respects existing configuration)\n    if multiprocessing.get_start_method(allow_none=True) is None:\n        try:\n            # 'fork' is available on Unix-like systems (Linux, macOS)\n            # On Windows, this will have no effect as 'spawn' is the only option\n            multiprocessing.set_start_method('fork', force=False)\n            logger.debug(\"Set multiprocessing start method to 'fork' for Python 3.14+ compatibility\")\n        except (ValueError, RuntimeError):\n            # Already set, not available on this platform, or context already created\n            pass\n\ndef pytest_addoption(parser):\n    \"\"\"Add custom command-line options for pytest.\n\n    Provides --datastore-path option for specifying custom datastore location.\n    Note: Cannot use -d short option as it's reserved by pytest for debug mode.\n    \"\"\"\n    parser.addoption(\n        \"--datastore-path\",\n        action=\"store\",\n        default=None,\n        help=\"Custom datastore path for tests\"\n    )\n\n@pytest.fixture(scope='session')\ndef datastore_path(tmp_path_factory, request):\n    \"\"\"Provide datastore path unique to this worker.\n\n    Supports custom path via --datastore-path/-d flag (mirrors main app).\n\n    CRITICAL for xdist isolation:\n    - Each WORKER gets its own directory\n    - Tests on same worker run SEQUENTIALLY and cleanup between tests\n    - No subdirectories needed since tests don't overlap on same worker\n    - Example: /tmp/test-datastore-gw0/ for worker gw0\n    \"\"\"\n    # Check for custom path first (mirrors main app's -d flag)\n    custom_path = request.config.getoption(\"--datastore-path\")\n    if custom_path:\n        # Ensure the directory exists\n        os.makedirs(custom_path, exist_ok=True)\n        logger.info(f\"Using custom datastore path: {custom_path}\")\n        return custom_path\n\n    # Otherwise use default tmp_path_factory logic\n    worker_id = getattr(request.config, 'workerinput', {}).get('workerid', 'master')\n    if worker_id == 'master':\n        path = tmp_path_factory.mktemp(\"test-datastore\")\n    else:\n        path = tmp_path_factory.mktemp(f\"test-datastore-{worker_id}\")\n    return str(path)\n\n\n@pytest.fixture(scope='function', autouse=True)\ndef prepare_test_function(live_server, datastore_path):\n    \"\"\"Prepare each test with complete isolation.\n\n    CRITICAL for xdist per-test isolation:\n    - Reuses the SAME datastore instance (so blueprint references stay valid)\n    - Clears all watches and state for a clean slate\n    - First watch will get uuid=\"first\"\n    \"\"\"\n    routes = [rule.rule for rule in live_server.app.url_map.iter_rules()]\n    if '/test-random-content-endpoint' not in routes:\n        logger.debug(\"Setting up test URL routes\")\n        new_live_server_setup(live_server)\n\n    # CRITICAL: Point app to THIS test's unique datastore directory\n    live_server.app.config['TEST_DATASTORE_PATH'] = datastore_path\n\n    # CRITICAL: Get datastore and stop it from writing stale data\n    datastore = live_server.app.config.get('DATASTORE')\n\n    # Clear the queue before starting the test to prevent state leakage\n    from changedetectionio.flask_app import update_q\n    while not update_q.empty():\n        try:\n            update_q.get_nowait()\n        except:\n            break\n\n    # Add test helper methods to the app for worker management\n    def set_workers(count):\n        \"\"\"Set the number of workers for testing - brutal shutdown, no delays\"\"\"\n        from changedetectionio import worker_pool\n        from changedetectionio.flask_app import update_q, notification_q\n\n        current_count = worker_pool.get_worker_count()\n\n       # Special case: Setting to 0 means shutdown all workers brutally\n        if count == 0:\n            logger.debug(f\"Brutally shutting down all {current_count} workers\")\n            worker_pool.shutdown_workers()\n            return {\n                'status': 'success',\n                'message': f'Shutdown all {current_count} workers',\n                'previous_count': current_count,\n                'current_count': 0\n            }\n\n        # Adjust worker count (no delays, no verification)\n        result = worker_pool.adjust_async_worker_count(\n            count,\n            update_q=update_q,\n            notification_q=notification_q,\n            app=live_server.app,\n            datastore=datastore\n        )\n\n        return result\n\n    def check_all_workers_alive(expected_count):\n        \"\"\"Check that all expected workers are alive\"\"\"\n        from changedetectionio import worker_pool\n        from changedetectionio.flask_app import update_q, notification_q\n        result = worker_pool.check_worker_health(\n            expected_count,\n            update_q=update_q,\n            notification_q=notification_q,\n            app=live_server.app,\n            datastore=datastore\n        )\n        assert result['status'] == 'healthy', f\"Workers not healthy: {result['message']}\"\n        return result\n\n    # Attach helper methods to app for easy test access\n    live_server.app.set_workers = set_workers\n    live_server.app.check_all_workers_alive = check_all_workers_alive\n\n\n\n\n    # CRITICAL: Clean up any files from previous tests\n    # This ensures a completely clean directory\n    cleanup(datastore_path)\n\n    # CRITICAL: Reload the EXISTING datastore instead of creating a new one\n    # This keeps blueprint references valid (they capture datastore at construction)\n    # reload_state() completely resets the datastore to a clean state\n\n    # Reload state with clean data (no default watches)\n    datastore.reload_state(\n        datastore_path=datastore_path,\n        include_default_watches=False,\n        version_tag=datastore.data.get('version_tag', '0.0.0')\n    )\n    live_server.app.secret_key = init_app_secret(datastore_path)\n    logger.debug(f\"prepare_test_function: Reloaded datastore at {hex(id(datastore))}\")\n    logger.debug(f\"prepare_test_function: Path {datastore.datastore_path}\")\n\n    yield\n\n    # Cleanup: Clear watches and queue after test\n    try:\n        from changedetectionio.flask_app import update_q\n        from pathlib import Path\n\n        # Clear the queue to prevent leakage to next test\n        while not update_q.empty():\n            try:\n                update_q.get_nowait()\n            except:\n                break\n\n        datastore.data['watching'] = {}\n\n        # Delete any old watch metadata JSON files\n        base_path = Path(datastore.datastore_path).resolve()\n        max_depth = 2\n\n        for file in base_path.rglob(\"*.json\"):\n            # Calculate depth relative to base path\n            depth = len(file.relative_to(base_path).parts) - 1\n\n            if depth <= max_depth and file.is_file():\n                file.unlink()\n\n    except Exception as e:\n        logger.warning(f\"Error during datastore cleanup: {e}\")\n\n\n# So the app can also know which test name it was\n@pytest.fixture(autouse=True)\ndef set_test_name(request):\n  \"\"\"Automatically set TEST_NAME env var for every test\"\"\"\n  test_name = request.node.name\n  os.environ['PYTEST_CURRENT_TEST'] = test_name\n  yield\n  # Cleanup if needed\n\n\n@pytest.fixture(scope='session')\ndef app(request, datastore_path):\n    \"\"\"Create application once per worker (session).\n\n    Note: Actual per-test isolation is handled by:\n    - prepare_test_function() recreates datastore and cleans directory\n    - All tests on same worker use same directory (cleaned between tests)\n    \"\"\"\n    # So they don't delay in fetching\n    os.environ[\"MINIMUM_SECONDS_RECHECK_TIME\"] = \"0\"\n    logger.debug(f\"Testing with datastore_path={datastore_path}\")\n    cleanup(datastore_path)\n\n    app_config = {'datastore_path': datastore_path, 'disable_checkver' : True}\n    cleanup(app_config['datastore_path'])\n\n    logger_level = 'TRACE'\n\n    logger.remove()\n    log_level_for_stdout = { 'DEBUG', 'SUCCESS' }\n    logger.configure(handlers=[\n        {\"sink\": sys.stdout, \"level\": logger_level,\n         \"filter\" : lambda record: record['level'].name in log_level_for_stdout},\n        {\"sink\": sys.stderr, \"level\": logger_level,\n         \"filter\": lambda record: record['level'].name not in log_level_for_stdout},\n        ])\n\n    datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], include_default_watches=False)\n    app = changedetection_app(app_config, datastore)\n\n    # Disable CSRF while running tests\n    app.config['WTF_CSRF_ENABLED'] = False\n    app.config['STOP_THREADS'] = True\n    # Store datastore_path so Flask routes can access it\n    app.config['TEST_DATASTORE_PATH'] = datastore_path\n\n    def teardown():\n        import threading\n        import time\n\n        # Stop all threads and services\n        datastore.stop_thread = True\n        app.config.exit.set()\n\n        # Shutdown workers gracefully before loguru cleanup\n        try:\n            from changedetectionio import worker_pool\n            worker_pool.shutdown_workers()\n        except Exception:\n            pass\n\n        # Stop socket server threads\n        try:\n            from changedetectionio.flask_app import socketio_server\n            if socketio_server and hasattr(socketio_server, 'shutdown'):\n                socketio_server.shutdown()\n        except Exception:\n            pass\n\n        # Get all active threads before cleanup\n        main_thread = threading.main_thread()\n        active_threads = [t for t in threading.enumerate() if t != main_thread and t.is_alive()]\n\n        # Wait for non-daemon threads to finish (with timeout)\n        timeout = 2.0  # 2 seconds max wait\n        start_time = time.time()\n\n        for thread in active_threads:\n            if not thread.daemon:\n                remaining_time = timeout - (time.time() - start_time)\n                if remaining_time > 0:\n                    logger.debug(f\"Waiting for non-daemon thread to finish: {thread.name}\")\n                    thread.join(timeout=remaining_time)\n                    if thread.is_alive():\n                        logger.warning(f\"Thread {thread.name} did not finish in time\")\n\n        # Give daemon threads a moment to finish their current work\n        time.sleep(0.2)\n\n        # Log any threads still running\n        remaining_threads = [t for t in threading.enumerate() if t != main_thread and t.is_alive()]\n        if remaining_threads:\n            logger.debug(f\"Threads still running after teardown: {[t.name for t in remaining_threads]}\")\n\n        # Remove all loguru handlers to prevent \"closed file\" errors\n        logger.remove()\n\n        # Cleanup files\n        cleanup(app_config['datastore_path'])\n\n       \n    request.addfinalizer(teardown)\n    yield app\n\n\n"
  },
  {
    "path": "changedetectionio/tests/custom_browser_url/__init__.py",
    "content": "# placeholder"
  },
  {
    "path": "changedetectionio/tests/custom_browser_url/test_custom_browser_url.py",
    "content": "#!/usr/bin/env python3\nimport os\n\nfrom flask import url_for\nfrom ..util import live_server_setup, wait_for_all_checks\n\ndef do_test(client, live_server, make_test_use_extra_browser=False):\n\n    # Grep for this string in the logs?\n    test_url = \"https://changedetection.io/ci-test.html?non-custom-default=true\"\n    # \"non-custom-default\" should not appear in the custom browser connection\n    custom_browser_name = 'custom browser URL'\n\n    # needs to be set and something like 'ws://127.0.0.1:3000'\n    assert os.getenv('PLAYWRIGHT_DRIVER_URL'), \"Needs PLAYWRIGHT_DRIVER_URL set for this test\"\n\n    #####################\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\"application-empty_pages_are_a_change\": \"\",\n              \"requests-time_between_check-minutes\": 180,\n              'application-fetch_backend': \"html_webdriver\",\n              'requests-extra_browsers-0-browser_connection_url': 'ws://sockpuppetbrowser-custom-url:3000',\n              'requests-extra_browsers-0-browser_name': custom_browser_name\n              },\n        follow_redirects=True\n    )\n\n    assert b\"Settings updated.\" in res.data\n\n    # Add our URL to the import page\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    if make_test_use_extra_browser:\n\n        # So the name should appear in the edit page under \"Request\" > \"Fetch Method\"\n        res = client.get(\n            url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n            follow_redirects=True\n        )\n        assert b'custom browser URL' in res.data\n\n        res = client.post(\n            url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n            data={\n                # 'run_customer_browser_url_tests.sh' will search for this string to know if we hit the right browser container or not\n                  \"url\": \"https://changedetection.io/ci-test.html?custom-browser-search-string=1\",\n                  \"tags\": \"\",\n                  \"headers\": \"\",\n                  'fetch_backend': f\"extra_browser_{custom_browser_name}\",\n                  'webdriver_js_execute_code': '',\n                  \"time_between_check_use_default\": \"y\"\n            },\n            follow_redirects=True\n        )\n\n        assert b\"Updated watch.\" in res.data\n        wait_for_all_checks(client)\n\n    # Force recheck\n    res = client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    assert b'Queued 1 watch for rechecking.' in res.data\n\n    wait_for_all_checks(client)\n\n    res = client.get(\n        url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"),\n        follow_redirects=True\n    )\n    assert b'cool it works' in res.data\n\n\n# Requires playwright to be installed\ndef test_request_via_custom_browser_url(client, live_server, measure_memory_usage, datastore_path):\n   #  live_server_setup(live_server) # Setup on conftest per function\n    # We do this so we can grep the logs of the custom container and see if the request actually went through that container\n    do_test(client, live_server, make_test_use_extra_browser=True)\n\n\ndef test_request_not_via_custom_browser_url(client, live_server, measure_memory_usage, datastore_path):\n   #  live_server_setup(live_server) # Setup on conftest per function\n    # We do this so we can grep the logs of the custom container and see if the request actually went through that container\n    do_test(client, live_server, make_test_use_extra_browser=False)\n"
  },
  {
    "path": "changedetectionio/tests/fetchers/__init__.py",
    "content": "\"\"\"Tests for the app.\"\"\"\n\n"
  },
  {
    "path": "changedetectionio/tests/fetchers/conftest.py",
    "content": "#!/usr/bin/env python3\n\nfrom .. import conftest\n"
  },
  {
    "path": "changedetectionio/tests/fetchers/test_content.py",
    "content": "#!/usr/bin/env python3\n\nimport time\nfrom flask import url_for\nimport os\nfrom ..util import live_server_setup, wait_for_all_checks\nimport logging\n\n\n# Requires playwright to be installed\ndef test_fetch_webdriver_content(client, live_server, measure_memory_usage, datastore_path):\n    #  live_server_setup(live_server) # Setup on conftest per function\n\n    #####################\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\n            \"application-empty_pages_are_a_change\": \"\",\n            \"requests-time_between_check-minutes\": 180,\n            'application-fetch_backend': \"html_webdriver\",\n            'application-ui-favicons_enabled': \"y\",\n        },\n        follow_redirects=True\n    )\n\n    assert b\"Settings updated.\" in res.data\n\n    # Add our URL to the import page\n    res = client.post(\n        url_for(\"imports.import_page\"),\n        data={\"urls\": \"https://changedetection.io/ci-test.html\"},\n        follow_redirects=True\n    )\n\n    assert b\"1 Imported\" in res.data\n    wait_for_all_checks(client)\n\n    res = client.get(\n        url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"),\n        follow_redirects=True\n    )\n    logging.getLogger().info(\"Looking for correct fetched HTML (text) from server\")\n    assert b'cool it works' in res.data\n\n    # Favicon scraper check, favicon only so far is fetched when in browser mode (not requests mode)\n    if os.getenv(\"PLAYWRIGHT_DRIVER_URL\"):\n        uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))\n        res = client.get(\n            url_for(\"watchlist.index\"),\n        )\n        # The UI can access it here\n        assert f'src=\"/static/favicon/{uuid}'.encode('utf8') in res.data\n\n        # Attempt to fetch it, make sure that works\n        res = client.get(url_for('static_content', group='favicon', filename=uuid))\n        assert res.status_code == 200\n        assert len(res.data) > 10\n\n        # Check the API also returns it\n        api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n        res = client.get(\n            url_for(\"watchfavicon\", uuid=uuid),\n            headers={'x-api-key': api_key}\n        )\n        assert res.status_code == 200\n        assert len(res.data) > 10\n\n    ##################### disable favicons check\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\n            \"requests-time_between_check-minutes\": 180,\n            'application-ui-favicons_enabled': \"\",\n            \"application-empty_pages_are_a_change\": \"\",\n        },\n        follow_redirects=True\n    )\n\n    assert b\"Settings updated.\" in res.data\n\n    res = client.get(\n        url_for(\"watchlist.index\"),\n    )\n    # The UI can access it here\n    assert f'src=\"/static/favicon'.encode('utf8') not in res.data\n"
  },
  {
    "path": "changedetectionio/tests/fetchers/test_custom_js_before_content.py",
    "content": "import os\nfrom flask import url_for\nfrom ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client\n\n\ndef test_execute_custom_js(client, live_server, measure_memory_usage, datastore_path):\n\n   #  live_server_setup(live_server) # Setup on conftest per function\n    assert os.getenv('PLAYWRIGHT_DRIVER_URL'), \"Needs PLAYWRIGHT_DRIVER_URL set for this test\"\n\n    test_url = url_for('test_interactive_html_endpoint', _external=True)\n    test_url = test_url.replace('localhost.localdomain', 'cdio')\n    test_url = test_url.replace('localhost', 'cdio')\n\n    res = client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": test_url, \"tags\": '', 'edit_and_watch_submit_button': 'Edit > Watch'},\n        follow_redirects=True\n    )\n\n    assert b\"Watch added in Paused state, saving will unpause\" in res.data\n\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\", unpause_on_save=1),\n        data={\n            \"url\": test_url,\n            \"tags\": \"\",\n            'fetch_backend': \"html_webdriver\",\n            'webdriver_js_execute_code': 'document.querySelector(\"button[name=test-button]\").click();',\n            'headers': \"testheader: yes\\buser-agent: MyCustomAgent\",\n            \"time_between_check_use_default\": \"y\",\n        },\n        follow_redirects=True\n    )\n    assert b\"unpaused\" in res.data\n    wait_for_all_checks(client)\n\n    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))\n    assert live_server.app.config['DATASTORE'].data['watching'][uuid].history_n >= 1, \"Watch history had atleast 1 (everything fetched OK)\"\n\n    assert b\"This text should be removed\" not in res.data\n\n    # Check HTML conversion detected and workd\n    res = client.get(\n        url_for(\"ui.ui_preview.preview_page\", uuid=uuid),\n        follow_redirects=True\n    )\n    assert b\"This text should be removed\" not in res.data\n    assert b\"I smell JavaScript because the button was pressed\" in res.data\n\n    assert b\"testheader: yes\" in res.data\n    assert b\"user-agent: mycustomagent\" in res.data\n\n    client.get(\n        url_for(\"ui.form_delete\", uuid=\"all\"),\n        follow_redirects=True\n    )"
  },
  {
    "path": "changedetectionio/tests/itemprop_test_examples/README.md",
    "content": "# A list of real world examples!\n\nAlways the price should be 666.66 for our tests\n\nsee test_restock_itemprop.py::test_special_prop_examples\n\n"
  },
  {
    "path": "changedetectionio/tests/itemprop_test_examples/a.txt",
    "content": "
    $155.55
    $155.55\n\n\n"
  },
  {
    "path": "changedetectionio/tests/plugins/test_processor.py",
    "content": "import time\n\nfrom flask import url_for\n\nfrom changedetectionio.tests.util import wait_for_all_checks\n\n\ndef test_check_plugin_processor(client, live_server, measure_memory_usage, datastore_path):\n    # requires os-int intelligence plugin installed (first basic one we test with)\n\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'OSINT Reconnaissance' in res.data, \"Must have the OSINT plugin installed at test time\"\n    assert b'' in res.data, \"But the first text_json_diff processor should always be selected by default in quick watch form\"\n\n    res = client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": 'http://127.0.0.1', \"tags\": '', 'processor': 'osint_recon'},\n        follow_redirects=True\n    )\n    assert b\"Watch added\" in res.data\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    wait_for_all_checks(client)\n\n    res = client.get(\n        url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"),\n        follow_redirects=True\n    )\n\n    assert b'Target: http://127.0.0.1' in res.data\n    assert b'DNSKEY Records' in res.data\n    wait_for_all_checks(client)\n\n\n    # Now change it to something that doesnt exist\n    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))\n    live_server.app.config['DATASTORE'].data['watching'][uuid]['processor'] = \"now_missing\"\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b\"Exception: Processor module\" in res.data and b'now_missing' in res.data, f'Should register that the plugin is missing for {uuid}'\n"
  },
  {
    "path": "changedetectionio/tests/proxy_list/__init__.py",
    "content": "\"\"\"Tests for the app.\"\"\"\n\n"
  },
  {
    "path": "changedetectionio/tests/proxy_list/conftest.py",
    "content": "#!/usr/bin/env python3\n\nfrom .. import conftest\n\n#def pytest_addoption(parser):\n#    parser.addoption(\"--url_suffix\", action=\"store\", default=\"identifier for request\")\n\n\n#def pytest_generate_tests(metafunc):\n#    # This is called for every test. Only get/set command line arguments\n#    # if the argument is specified in the list of test \"fixturenames\".\n#    option_value = metafunc.config.option.url_suffix\n#    if 'url_suffix' in metafunc.fixturenames and option_value is not None:\n#        metafunc.parametrize(\"url_suffix\", [option_value])"
  },
  {
    "path": "changedetectionio/tests/proxy_list/proxies.json-example",
    "content": "{\n  \"proxy-one\": {\n    \"label\": \"Proxy One\",\n    \"url\": \"http://squid-one:3128\"\n  },\n  \"proxy-two\": {\n    \"label\": \"Proxy Two\",\n    \"url\": \"http://squid-two:3128\"\n  }\n}\n"
  },
  {
    "path": "changedetectionio/tests/proxy_list/squid-auth.conf",
    "content": "acl localnet src 0.0.0.1-0.255.255.255  # RFC 1122 \"this\" network (LAN)\nacl localnet src 10.0.0.0/8             # RFC 1918 local private network (LAN)\nacl localnet src 100.64.0.0/10          # RFC 6598 shared address space (CGN)\nacl localnet src 169.254.0.0/16         # RFC 3927 link-local (directly plugged) machines\nacl localnet src 172.16.0.0/12          # RFC 1918 local private network (LAN)\nacl localnet src 192.168.0.0/16         # RFC 1918 local private network (LAN)\nacl localnet src fc00::/7               # RFC 4193 local private network range\nacl localnet src fe80::/10              # RFC 4291 link-local (directly plugged) machines\nacl localnet src 159.65.224.174\nacl SSL_ports port 443\nacl Safe_ports port 80          # http\nacl Safe_ports port 21          # ftp\nacl Safe_ports port 443         # https\nacl Safe_ports port 70          # gopher\nacl Safe_ports port 210         # wais\nacl Safe_ports port 1025-65535  # unregistered ports\nacl Safe_ports port 280         # http-mgmt\nacl Safe_ports port 488         # gss-http\nacl Safe_ports port 591         # filemaker\nacl Safe_ports port 777         # multiling http\nacl CONNECT method CONNECT\n\nhttp_access deny !Safe_ports\nhttp_access deny CONNECT !SSL_ports\n#http_access allow localhost manager\nhttp_access deny manager\n#http_access allow localhost\n#http_access allow localnet\n\nauth_param basic program /usr/lib/squid3/basic_ncsa_auth /etc/squid3/passwords\nauth_param basic realm proxy\nacl authenticated proxy_auth REQUIRED\nhttp_access allow authenticated\nhttp_access deny all\n\n\nhttp_port 3128\ncoredump_dir /var/spool/squid\nrefresh_pattern ^ftp:           1440    20%     10080\nrefresh_pattern ^gopher:        1440    0%      1440\nrefresh_pattern -i (/cgi-bin/|\\?) 0     0%      0\nrefresh_pattern \\/(Packages|Sources)(|\\.bz2|\\.gz|\\.xz)$ 0 0% 0 refresh-ims\nrefresh_pattern \\/Release(|\\.gpg)$ 0 0% 0 refresh-ims\nrefresh_pattern \\/InRelease$ 0 0% 0 refresh-ims\nrefresh_pattern \\/(Translation-.*)(|\\.bz2|\\.gz|\\.xz)$ 0 0% 0 refresh-ims\nrefresh_pattern .               0       20%     4320\nlogfile_rotate 0\n\n"
  },
  {
    "path": "changedetectionio/tests/proxy_list/squid-passwords.txt",
    "content": "test:$apr1$xvhFolTA$E/kz5/Rw1ewcyaSUdwqZs.\n"
  },
  {
    "path": "changedetectionio/tests/proxy_list/squid.conf",
    "content": "acl localnet src 0.0.0.1-0.255.255.255  # RFC 1122 \"this\" network (LAN)\nacl localnet src 10.0.0.0/8             # RFC 1918 local private network (LAN)\nacl localnet src 100.64.0.0/10          # RFC 6598 shared address space (CGN)\nacl localnet src 169.254.0.0/16         # RFC 3927 link-local (directly plugged) machines\nacl localnet src 172.16.0.0/12          # RFC 1918 local private network (LAN)\nacl localnet src 192.168.0.0/16         # RFC 1918 local private network (LAN)\nacl localnet src fc00::/7               # RFC 4193 local private network range\nacl localnet src fe80::/10              # RFC 4291 link-local (directly plugged) machines\nacl localnet src 159.65.224.174\nacl SSL_ports port 443\nacl Safe_ports port 80          # http\nacl Safe_ports port 21          # ftp\nacl Safe_ports port 443         # https\nacl Safe_ports port 70          # gopher\nacl Safe_ports port 210         # wais\nacl Safe_ports port 1025-65535  # unregistered ports\nacl Safe_ports port 280         # http-mgmt\nacl Safe_ports port 488         # gss-http\nacl Safe_ports port 591         # filemaker\nacl Safe_ports port 777         # multiling http\nacl CONNECT method CONNECT\n\nhttp_access deny !Safe_ports\nhttp_access deny CONNECT !SSL_ports\nhttp_access allow localhost manager\nhttp_access deny manager\nhttp_access allow localhost\nhttp_access allow localnet\nhttp_access deny all\nhttp_port 3128\ncoredump_dir /var/spool/squid\nrefresh_pattern ^ftp:           1440    20%     10080\nrefresh_pattern ^gopher:        1440    0%      1440\nrefresh_pattern -i (/cgi-bin/|\\?) 0     0%      0\nrefresh_pattern \\/(Packages|Sources)(|\\.bz2|\\.gz|\\.xz)$ 0 0% 0 refresh-ims\nrefresh_pattern \\/Release(|\\.gpg)$ 0 0% 0 refresh-ims\nrefresh_pattern \\/InRelease$ 0 0% 0 refresh-ims\nrefresh_pattern \\/(Translation-.*)(|\\.bz2|\\.gz|\\.xz)$ 0 0% 0 refresh-ims\nrefresh_pattern .               0       20%     4320\nlogfile_rotate 0\n\n"
  },
  {
    "path": "changedetectionio/tests/proxy_list/test_multiple_proxy.py",
    "content": "#!/usr/bin/env python3\n\nimport os\nfrom flask import url_for\nfrom ..util import live_server_setup, wait_for_all_checks\n\n\ndef test_preferred_proxy(client, live_server, measure_memory_usage, datastore_path):\n   #  live_server_setup(live_server) # Setup on conftest per function\n    url = \"http://chosen.changedetection.io\"\n\n\n    res = client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": url, \"tags\": '', 'edit_and_watch_submit_button': 'Edit > Watch'},\n        follow_redirects=True\n    )\n    assert b\"Watch added in Paused state, saving will unpause\" in res.data\n\n    wait_for_all_checks(client)\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\", unpause_on_save=1),\n        data={\n                \"include_filters\": \"\",\n                \"fetch_backend\": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests',\n                \"headers\": \"\",\n                \"proxy\": \"proxy-two\",\n                \"tags\": \"\",\n                \"url\": url,\n                \"time_between_check_use_default\": \"y\",\n              },\n        follow_redirects=True\n    )\n    assert b\"unpaused\" in res.data\n    wait_for_all_checks(client)\n    # Now the request should appear in the second-squid logs\n"
  },
  {
    "path": "changedetectionio/tests/proxy_list/test_noproxy.py",
    "content": "#!/usr/bin/env python3\n\nimport time\nfrom flask import url_for\nfrom ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client\n\n\ndef test_noproxy_option(client, live_server, measure_memory_usage, datastore_path):\n   #  live_server_setup(live_server) # Setup on conftest per function\n    # Run by run_proxy_tests.sh\n    # Call this URL then scan the containers that it never went through them\n    url = \"http://noproxy.changedetection.io\"\n\n    # Should only be available when a proxy is setup\n    res = client.get(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\", unpause_on_save=1))\n    assert b'No proxy' not in res.data\n\n    # Setup a proxy\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\n            \"requests-time_between_check-minutes\": 180,\n            \"application-ignore_whitespace\": \"y\",\n            \"application-fetch_backend\": \"html_requests\",\n            \"requests-extra_proxies-0-proxy_name\": \"custom-one-proxy\",\n            \"requests-extra_proxies-0-proxy_url\": \"http://test:awesome@squid-one:3128\",\n            \"requests-extra_proxies-1-proxy_name\": \"custom-two-proxy\",\n            \"requests-extra_proxies-1-proxy_url\": \"http://test:awesome@squid-two:3128\",\n            \"requests-extra_proxies-2-proxy_name\": \"custom-proxy\",\n            \"requests-extra_proxies-2-proxy_url\": \"http://test:awesome@squid-custom:3128\",\n        },\n        follow_redirects=True\n    )\n\n    assert b\"Settings updated.\" in res.data\n\n    # Should be available as an option\n    res = client.get(\n        url_for(\"settings.settings_page\", unpause_on_save=1))\n    assert b'No proxy' in res.data\n\n\n    # This will add it paused\n    res = client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": url, \"tags\": '', 'edit_and_watch_submit_button': 'Edit > Watch'},\n        follow_redirects=True\n    )\n    assert b\"Watch added in Paused state, saving will unpause\" in res.data\n    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))\n    res = client.get(\n        url_for(\"ui.ui_edit.edit_page\", uuid=uuid, unpause_on_save=1))\n    assert b'No proxy' in res.data\n\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=uuid, unpause_on_save=1),\n        data={\n                \"include_filters\": \"\",\n                \"fetch_backend\": \"html_requests\",\n                \"headers\": \"\",\n                \"proxy\": \"no-proxy\",\n                \"tags\": \"\",\n                \"url\": url,\n                \"time_between_check_use_default\": \"y\",\n              },\n        follow_redirects=True\n    )\n    assert b\"unpaused\" in res.data\n    wait_for_all_checks(client)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n    # Now the request should NOT appear in the second-squid logs (handled by the run_test_proxies.sh script)\n\n    # Prove that it actually checked\n\n    assert live_server.app.config['DATASTORE'].data['watching'][uuid]['last_checked'] != 0\n\n"
  },
  {
    "path": "changedetectionio/tests/proxy_list/test_proxy.py",
    "content": "#!/usr/bin/env python3\n\nimport time\nfrom flask import url_for\nfrom ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client\n\n# just make a request, we will grep in the docker logs to see it actually got called\ndef test_check_basic_change_detection_functionality(client, live_server, measure_memory_usage, datastore_path):\n   #  live_server_setup(live_server) # Setup on conftest per function\n    res = client.post(\n        url_for(\"imports.import_page\"),\n        # Because a URL wont show in squid/proxy logs due it being SSLed\n        # Use plain HTTP or a specific domain-name here\n        data={\"urls\": \"http://one.changedetection.io\"},\n        follow_redirects=True\n    )\n\n    assert b\"1 Imported\" in res.data\n    wait_for_all_checks(client)\n"
  },
  {
    "path": "changedetectionio/tests/proxy_list/test_proxy_noconnect.py",
    "content": "#!/usr/bin/env python3\n\nfrom flask import url_for\nfrom ..util import live_server_setup, wait_for_all_checks\nimport os\nfrom ... import strtobool\n\n\n# Just to be sure the UI outputs the right error message on proxy connection failed\n# docker run -p 4444:4444 --rm --shm-size=\"2g\"  selenium/standalone-chrome:4\n# PLAYWRIGHT_DRIVER_URL=ws://127.0.0.1:3000 pytest tests/proxy_list/test_proxy_noconnect.py\n# FAST_PUPPETEER_CHROME_FETCHER=True PLAYWRIGHT_DRIVER_URL=ws://127.0.0.1:3000 pytest tests/proxy_list/test_proxy_noconnect.py\n# WEBDRIVER_URL=http://127.0.0.1:4444/wd/hub pytest tests/proxy_list/test_proxy_noconnect.py\n\ndef test_proxy_noconnect_custom(client, live_server, measure_memory_usage, datastore_path):\n   #  live_server_setup(live_server) # Setup on conftest per function\n\n    # Goto settings, add our custom one\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\n            \"requests-time_between_check-minutes\": 180,\n            \"application-ignore_whitespace\": \"y\",\n            \"application-fetch_backend\": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') or os.getenv(\"WEBDRIVER_URL\") else 'html_requests',\n            \"requests-extra_proxies-0-proxy_name\": \"custom-test-proxy\",\n            # test:awesome is set in tests/proxy_list/squid-passwords.txt\n            \"requests-extra_proxies-0-proxy_url\": \"http://127.0.0.1:3128\",\n        },\n        follow_redirects=True\n    )\n\n    assert b\"Settings updated.\" in res.data\n\n    test_url = \"https://changedetection.io\"\n    res = client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": test_url, \"tags\": '', 'edit_and_watch_submit_button': 'Edit > Watch'},\n        follow_redirects=True\n    )\n\n    assert b\"Watch added in Paused state, saving will unpause\" in res.data\n\n    options = {\n        \"url\": test_url,\n        \"fetch_backend\": \"html_webdriver\" if os.getenv('PLAYWRIGHT_DRIVER_URL') or os.getenv(\"WEBDRIVER_URL\") else \"html_requests\",\n        \"proxy\": \"ui-0custom-test-proxy\",\n        \"time_between_check_use_default\": \"y\",\n    }\n\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\", unpause_on_save=1),\n        data=options,\n        follow_redirects=True\n    )\n    assert b\"unpaused\" in res.data\n    import time\n    wait_for_all_checks(client)\n\n    # Requests default\n    check_string = b'Cannot connect to proxy'\n\n    if os.getenv('PLAYWRIGHT_DRIVER_URL') or strtobool(os.getenv('FAST_PUPPETEER_CHROME_FETCHER', 'False')) or os.getenv(\"WEBDRIVER_URL\"):\n        check_string = b'ERR_PROXY_CONNECTION_FAILED'\n\n\n    res = client.get(url_for(\"watchlist.index\"))\n    #with open(\"/tmp/debug.html\", 'wb') as f:\n    #    f.write(res.data)\n    assert check_string in res.data\n"
  },
  {
    "path": "changedetectionio/tests/proxy_list/test_select_custom_proxy.py",
    "content": "#!/usr/bin/env python3\n\nimport time\nfrom flask import url_for\nfrom ..util import live_server_setup, wait_for_all_checks\nimport os\n\n# just make a request, we will grep in the docker logs to see it actually got called\ndef test_select_custom(client, live_server, measure_memory_usage, datastore_path):\n   #  live_server_setup(live_server) # Setup on conftest per function\n\n    # Goto settings, add our custom one\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\n            \"requests-time_between_check-minutes\": 180,\n            \"application-ignore_whitespace\": \"y\",\n            \"application-fetch_backend\": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests',\n            \"requests-extra_proxies-0-proxy_name\": \"custom-test-proxy\",\n            # test:awesome is set in tests/proxy_list/squid-passwords.txt\n            \"requests-extra_proxies-0-proxy_url\": \"http://test:awesome@squid-custom:3128\",\n        },\n        follow_redirects=True\n    )\n\n    assert b\"Settings updated.\" in res.data\n\n    res = client.post(\n        url_for(\"imports.import_page\"),\n        # Because a URL wont show in squid/proxy logs due it being SSLed\n        # Use plain HTTP or a specific domain-name here\n        data={\"urls\": \"https://changedetection.io/CHANGELOG.txt\"},\n        follow_redirects=True\n    )\n\n    assert b\"1 Imported\" in res.data\n    wait_for_all_checks(client)\n\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'Proxy Authentication Required' not in res.data\n\n    res = client.get(\n        url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"),\n        follow_redirects=True\n    )\n    # We should see something via proxy\n    assert b' - 0.' in res.data\n\n    #\n    # Now we should see the request in the container logs for \"squid-squid-custom\" because it will be the only default\n\n\ndef test_custom_proxy_validation(client, live_server, measure_memory_usage, datastore_path):\n    #  live_server_setup(live_server) # Setup on conftest per function\n\n    # Goto settings, add our custom one\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\n            \"requests-time_between_check-minutes\": 180,\n            \"application-ignore_whitespace\": \"y\",\n            \"application-fetch_backend\": 'html_requests',\n            \"requests-extra_proxies-0-proxy_name\": \"custom-test-proxy\",\n            \"requests-extra_proxies-0-proxy_url\": \"xxxxhtt/333??p://test:awesome@squid-custom:3128\",\n        },\n        follow_redirects=True\n    )\n\n    assert b\"Settings updated.\" not in res.data\n    assert b'Proxy URLs must start with' in res.data\n\n\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\n            \"requests-time_between_check-minutes\": 180,\n            \"application-ignore_whitespace\": \"y\",\n            \"application-fetch_backend\": 'html_requests',\n            \"requests-extra_proxies-0-proxy_name\": \"custom-test-proxy\",\n            \"requests-extra_proxies-0-proxy_url\": \"https://\",\n        },\n        follow_redirects=True\n    )\n\n    assert b\"Settings updated.\" not in res.data\n    assert b\"Invalid URL.\" in res.data\n    "
  },
  {
    "path": "changedetectionio/tests/proxy_socks5/proxies.json-example",
    "content": "{\n  \"socks5proxy\": {\n    \"label\": \"socks5proxy\",\n    \"url\": \"socks5://proxy_user123:proxy_pass123@socks5proxy:1080\"\n  }\n}\n"
  },
  {
    "path": "changedetectionio/tests/proxy_socks5/proxies.json-example-noauth",
    "content": "{\n  \"socks5proxy\": {\n    \"label\": \"socks5proxy\",\n    \"url\": \"socks5://socks5proxy-noauth:1080\"\n  }\n}\n"
  },
  {
    "path": "changedetectionio/tests/proxy_socks5/test_socks5_proxy.py",
    "content": "#!/usr/bin/env python3\nimport json\nimport os\nfrom flask import url_for\nfrom changedetectionio.tests.util import live_server_setup, wait_for_all_checks, extract_UUID_from_client, delete_all_watches\n\n\ndef set_response(datastore_path):\n    import time\n    data = \"\"\"\n       \n     
    Awesome, you made it
    \n     yeah the socks request worked\n     \n     \n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(data)\n    time.sleep(1)\n\ndef test_socks5(client, live_server, measure_memory_usage, datastore_path):\n   #  live_server_setup(live_server) # Setup on conftest per function\n    set_response(datastore_path)\n\n    # Setup a proxy\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\n            \"requests-time_between_check-minutes\": 180,\n            \"application-ignore_whitespace\": \"y\",\n            \"application-fetch_backend\": \"html_requests\",\n            # set in .github/workflows/test-only.yml\n            \"requests-extra_proxies-0-proxy_url\": \"socks5://proxy_user123:proxy_pass123@socks5proxy:1080\",\n            \"requests-extra_proxies-0-proxy_name\": \"socks5proxy\",\n        },\n        follow_redirects=True\n    )\n\n    assert b\"Settings updated.\" in res.data\n\n    # Because the socks server should connect back to us\n    test_url = url_for('test_endpoint', _external=True) + f\"?socks-test-tag={os.getenv('SOCKSTEST', '')}\"\n    test_url = test_url.replace('localhost.localdomain', 'cdio')\n    test_url = test_url.replace('localhost', 'cdio')\n\n    res = client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": test_url, \"tags\": '', 'edit_and_watch_submit_button': 'Edit > Watch'},\n        follow_redirects=True\n    )\n    assert b\"Watch added in Paused state, saving will unpause\" in res.data\n\n    res = client.get(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\", unpause_on_save=1),\n    )\n    # check the proxy is offered as expected\n    assert b'ui-0socks5proxy' in res.data\n\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\", unpause_on_save=1),\n        data={\n            \"include_filters\": \"\",\n            \"fetch_backend\": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests',\n            \"headers\": \"\",\n            \"proxy\": \"ui-0socks5proxy\",\n            \"tags\": \"\",\n            \"url\": test_url,\n            \"time_between_check_use_default\": \"y\",\n        },\n        follow_redirects=True\n    )\n    assert b\"unpaused\" in res.data\n    wait_for_all_checks(client)\n\n    res = client.get(\n        url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"),\n        follow_redirects=True\n    )\n\n    # Should see the proper string\n    assert \"Awesome, you made it\".encode('utf-8') in res.data\n\n    # PROXY CHECKER WIDGET CHECK - this needs more checking\n    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))\n\n    res = client.get(\n        url_for(\"check_proxies.start_check\", uuid=uuid),\n        follow_redirects=True\n    )\n    # It's probably already finished super fast :(\n    #assert b\"RUNNING\" in res.data\n    \n    wait_for_all_checks(client)\n    res = client.get(\n        url_for(\"check_proxies.get_recheck_status\", uuid=uuid),\n        follow_redirects=True\n    )\n    assert b\"OK\" in res.data\n\n    delete_all_watches(client)\n\n"
  },
  {
    "path": "changedetectionio/tests/proxy_socks5/test_socks5_proxy_sources.py",
    "content": "#!/usr/bin/env python3\nimport os\nfrom flask import url_for\nfrom changedetectionio.tests.util import live_server_setup, wait_for_all_checks\n\n\ndef set_response(datastore_path):\n    import time\n    data = \"\"\"\n       \n     
    Awesome, you made it
    \n     yeah the socks request worked\n     \n     \n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(data)\n    time.sleep(1)\n\n# should be proxies.json mounted from run_proxy_tests.sh already\n# -v `pwd`/tests/proxy_socks5/proxies.json-example:/app/changedetectionio/test-datastore/proxies.json\ndef test_socks5_from_proxiesjson_file(client, live_server, measure_memory_usage, datastore_path):\n   #  live_server_setup(live_server) # Setup on conftest per function\n    set_response(datastore_path)\n    # Because the socks server should connect back to us\n    test_url = url_for('test_endpoint', _external=True) + f\"?socks-test-tag={os.getenv('SOCKSTEST', '')}\"\n    test_url = test_url.replace('localhost.localdomain', 'cdio')\n    test_url = test_url.replace('localhost', 'cdio')\n\n    res = client.get(url_for(\"settings.settings_page\"))\n    assert b'name=\"requests-proxy\" type=\"radio\" value=\"socks5proxy\"' in res.data\n\n    res = client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": test_url, \"tags\": '', 'edit_and_watch_submit_button': 'Edit > Watch'},\n        follow_redirects=True\n    )\n    assert b\"Watch added in Paused state, saving will unpause\" in res.data\n\n    res = client.get(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\", unpause_on_save=1),\n    )\n    # check the proxy is offered as expected\n    assert b'name=\"proxy\" type=\"radio\" value=\"socks5proxy\"' in res.data\n\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\", unpause_on_save=1),\n        data={\n            \"include_filters\": \"\",\n            \"fetch_backend\": 'html_webdriver' if os.getenv('PLAYWRIGHT_DRIVER_URL') else 'html_requests',\n            \"headers\": \"\",\n            \"proxy\": \"socks5proxy\",\n            \"tags\": \"\",\n            \"url\": test_url,\n            \"time_between_check_use_default\": \"y\",\n        },\n        follow_redirects=True\n    )\n    assert b\"unpaused\" in res.data\n    wait_for_all_checks(client)\n\n    res = client.get(\n        url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"),\n        follow_redirects=True\n    )\n\n    # Should see the proper string\n    assert \"Awesome, you made it\".encode('utf-8') in res.data\n"
  },
  {
    "path": "changedetectionio/tests/restock/__init__.py",
    "content": "\"\"\"Tests for the app.\"\"\"\n\n"
  },
  {
    "path": "changedetectionio/tests/restock/conftest.py",
    "content": "#!/usr/bin/env python3\n\nfrom .. import conftest\n"
  },
  {
    "path": "changedetectionio/tests/restock/test_restock.py",
    "content": "#!/usr/bin/env python3\nimport os\nimport time\nfrom flask import url_for\nfrom ..util import live_server_setup, wait_for_all_checks, extract_UUID_from_client, wait_for_notification_endpoint_output\nfrom changedetectionio.notification import (\n    default_notification_body,\n    default_notification_format,\n    default_notification_title,\n    valid_notification_formats,\n)\n\n\ndef set_original_response(datastore_path):\n    test_return_data = \"\"\"\n       \n       \n       \n     Some initial text
    \n     
    Which is across multiple lines
    \n     
    \n     So let's see what happens.  
    \n     
    price: $10.99
    \n     
    Out of stock
    \n     \n     \n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n    return None\n\n\n\ndef set_back_in_stock_response(datastore_path):\n    test_return_data = \"\"\"\n       \n     Some initial text
    \n     
    Which is across multiple lines
    \n     
    \n     So let's see what happens.  
    \n     
    price: $10.99
    \n     
    Available!
    \n     \n     \n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n    return None\n\n# Add a site in paused mode, add an invalid filter, we should still have visual selector data ready\ndef test_restock_detection(client, live_server, measure_memory_usage, datastore_path):\n\n    set_original_response(datastore_path=datastore_path)\n    #assert os.getenv('PLAYWRIGHT_DRIVER_URL'), \"Needs PLAYWRIGHT_DRIVER_URL set for this test\"\n   #  live_server_setup(live_server) # Setup on conftest per function\n    #####################\n    notification_url = url_for('test_notification_endpoint', _external=True).replace('http://localhost', 'http://changedet').replace('http', 'json')\n\n\n    #####################\n    # Set this up for when we remove the notification from the watch, it should fallback with these details\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\"application-notification_urls\": notification_url,\n              \"application-notification_title\": \"fallback-title \"+default_notification_title,\n              \"application-notification_body\": \"fallback-body \"+default_notification_body,\n              \"application-notification_format\": default_notification_format,\n              \"requests-time_between_check-minutes\": 180,\n              'application-fetch_backend': \"html_webdriver\"},\n        follow_redirects=True\n    )\n    # Add our URL to the import page, because the docker container (playwright/selenium) wont be able to connect to our usual test url\n    test_url = url_for('test_endpoint', _external=True).replace('http://localhost', 'http://changedet')\n\n\n    client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": test_url, \"tags\": '', 'processor': 'restock_diff'},\n        follow_redirects=True\n    )\n\n    # Is it correctly show as NOT in stock?\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'processor-restock_diff' in res.data # Should have saved in restock mode\n    assert b'not-in-stock' in res.data # should be out of stock\n\n    # Is it correctly shown as in stock\n    set_back_in_stock_response(datastore_path)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'not-in-stock' not in res.data\n\n    # We should have a notification\n    notification_file = os.path.join(datastore_path, \"notification.txt\")\n    wait_for_notification_endpoint_output(datastore_path=datastore_path)\n    assert os.path.isfile(notification_file), \"Notification received\"\n    os.unlink(notification_file)\n\n    # Default behaviour is to only fire notification when it goes OUT OF STOCK -> IN STOCK\n    # So here there should be no file, because we go IN STOCK -> OUT OF STOCK\n    set_original_response(datastore_path=datastore_path)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n    time.sleep(5)\n    assert not os.path.isfile(notification_file), \"No notification should have fired when it went OUT OF STOCK by default\"\n\n    # BUT we should see that it correctly shows \"not in stock\"\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'not-in-stock' in res.data, \"Correctly showing NOT IN STOCK in the list after it changed from IN STOCK\"\n\n"
  },
  {
    "path": "changedetectionio/tests/smtp/smtp-test-server.py",
    "content": "#!/usr/bin/env python3\nimport threading\nimport time\nfrom aiosmtpd.controller import Controller\nfrom flask import Flask, Response\nfrom email import message_from_bytes\nfrom email.policy import default\n\n# Accept a SMTP message and offer a way to retrieve the last message via HTTP\n\nlast_received_message = b\"SMTP Test Server - Nothing received yet.\"\nactive_smtp_connections = 0\nsmtp_lock = threading.Lock()\n\n\nclass CustomSMTPHandler:\n    async def handle_DATA(self, server, session, envelope):\n        global last_received_message, active_smtp_connections\n\n        with smtp_lock:\n            active_smtp_connections += 1\n\n        try:\n            last_received_message = envelope.content\n            print('Receiving message from:', session.peer)\n            print('Message addressed from:', envelope.mail_from)\n            print('Message addressed to  :', envelope.rcpt_tos)\n            print('Message length        :', len(envelope.content))\n            print('*******************************')\n            print(envelope.content.decode('utf8'))\n            print('*******************************')\n\n            # Parse the email message\n            msg = message_from_bytes(envelope.content, policy=default)\n            with open('/tmp/last.eml', 'wb') as f:\n                f.write(envelope.content)\n\n            # Write parts to files based on content type\n            if msg.is_multipart():\n                for part in msg.walk():\n                    content_type = part.get_content_type()\n                    payload = part.get_payload(decode=True)\n\n                    if payload:\n                        if content_type == 'text/plain':\n                            with open('/tmp/last.txt', 'wb') as f:\n                                f.write(payload)\n                            print(f'Written text/plain part to /tmp/last.txt')\n                        elif content_type == 'text/html':\n                            with open('/tmp/last.html', 'wb') as f:\n                                f.write(payload)\n                            print(f'Written text/html part to /tmp/last.html')\n            else:\n                # Single part message\n                content_type = msg.get_content_type()\n                payload = msg.get_payload(decode=True)\n\n                if payload:\n                    if content_type == 'text/plain' or content_type.startswith('text/'):\n                        with open('/tmp/last.txt', 'wb') as f:\n                            f.write(payload)\n                        print(f'Written single part message to /tmp/last.txt')\n\n            return '250 Message accepted for delivery'\n        finally:\n            with smtp_lock:\n                active_smtp_connections -= 1\n\n\n# Simple Flask HTTP server to echo back the last SMTP message\napp = Flask(__name__)\n\n\n@app.route('/')\ndef echo_last_message():\n    global last_received_message, active_smtp_connections\n\n    # Wait for any in-progress SMTP connections to complete\n    max_wait = 5  # Maximum 5 seconds\n    wait_interval = 0.05  # Check every 50ms\n    elapsed = 0\n\n    while elapsed < max_wait:\n        with smtp_lock:\n            if active_smtp_connections == 0:\n                break\n        time.sleep(wait_interval)\n        elapsed += wait_interval\n\n    return Response(last_received_message, mimetype='text/plain')\n\n\ndef run_flask():\n    app.run(host='0.0.0.0', port=11080, debug=False, use_reloader=False)\n\n\nif __name__ == \"__main__\":\n    # Start the SMTP server\n    controller = Controller(CustomSMTPHandler(), hostname='0.0.0.0', port=11025)\n    controller.start()\n\n    # Start the HTTP server in a separate thread\n    flask_thread = threading.Thread(target=run_flask, daemon=True)\n    flask_thread.start()\n\n    # Keep the main thread alive\n    try:\n        flask_thread.join()\n    except KeyboardInterrupt:\n        print(\"Shutting down...\")\n"
  },
  {
    "path": "changedetectionio/tests/smtp/test_notification_smtp.py",
    "content": "import time\nfrom flask import url_for\nfrom email import message_from_string\nfrom email.policy import default as email_policy\n\nfrom changedetectionio.diff import HTML_REMOVED_STYLE, HTML_ADDED_STYLE, HTML_CHANGED_STYLE, REMOVED_PLACEMARKER_OPEN, \\\n    CHANGED_PLACEMARKER_OPEN, ADDED_PLACEMARKER_OPEN\nfrom changedetectionio.notification_service import NotificationContextData\nfrom changedetectionio.tests.util import set_original_response, set_modified_response, set_more_modified_response, live_server_setup, \\\n    wait_for_all_checks, \\\n    set_longer_modified_response, delete_all_watches\n\nimport logging\n\n\n# NOTE - RELIES ON mailserver as hostname running, see github build recipes\nsmtp_test_server = 'mailserver'\n\nALL_MARKUP_TOKENS = ''.join(f\"TOKEN: '{t}'\\n{{{{{t}}}}}\\n\" for t in NotificationContextData().keys())\n\nfrom changedetectionio.notification import (\n    default_notification_body,\n    default_notification_format,\n    default_notification_title,\n    valid_notification_formats,\n)\n\n\n\ndef get_last_message_from_smtp_server():\n    import requests\n    time.sleep(1) # wait for any smtp connects to die off\n    port = 11080  # HTTP server port number\n    # Make HTTP GET request to Flask server\n    response = requests.get(f'http://{smtp_test_server}:{port}/')\n    data = response.text\n    logging.info(\"get_last_message_from_smtp_server..\")\n    logging.info(data)\n    return data\n\n\n# Requires running the test SMTP server\n\ndef test_check_notification_email_formats_default_HTML(client, live_server, measure_memory_usage, datastore_path):\n    ##  live_server_setup(live_server) # Setup on conftest per function\n    set_original_response(datastore_path=datastore_path)\n\n\n    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'\n\n    #####################\n    # Set this up for when we remove the notification from the watch, it should fallback with these details\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\"application-notification_urls\": notification_url,\n              \"application-notification_title\": \"fallback-title \" + default_notification_title,\n              \"application-notification_body\": \"some text\\nfallback-body
     \" + default_notification_body,\n              \"application-notification_format\": 'html',\n              \"requests-time_between_check-minutes\": 180,\n              'application-fetch_backend': \"html_requests\"},\n        follow_redirects=True\n    )\n    assert b\"Settings updated.\" in res.data\n\n    # Add a watch and trigger a HTTP POST\n    test_url = url_for('test_endpoint', _external=True)\n    res = client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": test_url, \"tags\": 'nice one'},\n        follow_redirects=True\n    )\n\n    assert b\"Watch added\" in res.data\n\n    wait_for_all_checks(client)\n    set_longer_modified_response(datastore_path=datastore_path)\n    time.sleep(2)\n\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    time.sleep(3)\n\n    msg_raw = get_last_message_from_smtp_server()\n    assert len(msg_raw) >= 1\n\n    # Parse the email properly using Python's email library\n    msg = message_from_string(msg_raw, policy=email_policy)\n\n    # The email should have two bodies (multipart/alternative with text/plain and text/html)\n    assert msg.is_multipart()\n    assert msg.get_content_type() == 'multipart/alternative'\n\n    # Get the parts\n    parts = list(msg.iter_parts())\n    assert len(parts) == 2\n\n    # First part should be text/plain (the auto-generated plaintext version)\n    text_part = parts[0]\n    assert text_part.get_content_type() == 'text/plain'\n    text_content = text_part.get_content()\n    assert '(added) So let\\'s see what happens.\\r\\n' in text_content  # The plaintext part\n    assert 'fallback-body\\r\\n' in text_content  # The plaintext part\n\n    # Second part should be text/html\n    html_part = parts[1]\n    assert html_part.get_content_type() == 'text/html'\n    html_content = html_part.get_content()\n    assert 'some text
    ' in html_content  # We converted \\n from the notification body\n    assert 'fallback-body
    ' in html_content  # kept the original 
    \n    assert '(added) So let\\'s see what happens.
    ' in html_content  # the html part\n    delete_all_watches(client)\n\n\ndef test_check_notification_plaintext_format(client, live_server, measure_memory_usage, datastore_path):\n    set_original_response(datastore_path=datastore_path)\n\n\n    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'\n\n    #####################\n    # Set this up for when we remove the notification from the watch, it should fallback with these details\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\"application-notification_urls\": notification_url,\n              \"application-notification_title\": \"fallback-title {{watch_title}}  {{ diff_added.splitlines()[0] if diff_added else 'diff added didnt split' }}  \" + default_notification_title,\n              \"application-notification_body\": f\"some text\\n\" + default_notification_body + f\"\\nMore output test\\n{ALL_MARKUP_TOKENS}\",\n              \"application-notification_format\": 'text',\n              \"requests-time_between_check-minutes\": 180,\n              'application-fetch_backend': \"html_requests\"},\n        follow_redirects=True\n    )\n\n    assert b\"Settings updated.\" in res.data\n\n    # Add a watch and trigger a HTTP POST\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    time.sleep(2)\n\n    set_longer_modified_response(datastore_path=datastore_path)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    time.sleep(3)\n\n    msg_raw = get_last_message_from_smtp_server()\n    assert len(msg_raw) >= 1\n    #time.sleep(60)\n    # Parse the email properly using Python's email library\n    msg = message_from_string(msg_raw, policy=email_policy)\n    # Subject/title got marked up\n    subject = msg['subject']\n    # Subject should always be plaintext and never marked up to anything else\n    assert REMOVED_PLACEMARKER_OPEN not in subject\n    assert CHANGED_PLACEMARKER_OPEN not in subject\n    assert ADDED_PLACEMARKER_OPEN not in subject\n    assert 'diff added didnt split' not in subject\n    assert '(changed) Which is across' in subject\n    assert 'PLACEMARKER' not in subject\n\n    # The email should be plain text only (not multipart)\n    assert not msg.is_multipart()\n    assert msg.get_content_type() == 'text/plain'\n\n    # Get the plain text content\n    text_content = msg.get_content()\n    assert '(added) So let\\'s see what happens.\\r\\n' in text_content  # The plaintext part\n\n    # Should NOT contain HTML\n    assert '
    ' not in text_content  # We should not have HTML in plain text\n    delete_all_watches(client)\n\n\n\ndef test_check_notification_html_color_format(client, live_server, measure_memory_usage, datastore_path):\n    set_original_response(datastore_path=datastore_path)\n\n\n    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'\n\n    #####################\n    # Set this up for when we remove the notification from the watch, it should fallback with these details\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\"application-notification_urls\": notification_url,\n              \"application-notification_title\": \"fallback-title {{watch_title}} - diff_added_lines_test : '{{ diff_added.splitlines()[0] if diff_added else 'diff added didnt split' }}' \" + default_notification_title,\n              \"application-notification_body\": f\"some text\\n{default_notification_body}\\nMore output test\\n{ALL_MARKUP_TOKENS}\",\n              \"application-notification_format\": 'htmlcolor',\n              \"requests-time_between_check-minutes\": 180,\n              'application-fetch_backend': \"html_requests\"},\n        follow_redirects=True\n    )\n\n    assert b\"Settings updated.\" in res.data\n\n    # Add a watch and trigger a HTTP POST\n    test_url = url_for('test_endpoint', _external=True)\n    res = client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": test_url, \"tags\": 'nice one'},\n        follow_redirects=True\n    )\n\n    assert b\"Watch added\" in res.data\n\n    wait_for_all_checks(client)\n    set_longer_modified_response(datastore_path=datastore_path)\n    time.sleep(2)\n\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    time.sleep(3)\n\n    msg_raw = get_last_message_from_smtp_server()\n    assert len(msg_raw) >= 1\n\n    # Parse the email properly using Python's email library\n    msg = message_from_string(msg_raw, policy=email_policy)\n    # Subject/title got marked up\n    subject = msg['subject']\n    # Subject should always be plaintext and never marked up to anything else\n    assert REMOVED_PLACEMARKER_OPEN not in subject\n    assert CHANGED_PLACEMARKER_OPEN not in subject\n    assert ADDED_PLACEMARKER_OPEN not in subject\n    assert 'diff added didnt split' not in subject\n    assert '(changed) Which is across' in subject\n    assert 'PLACEMARKER' not in subject\n    assert 'head title' in subject\n    assert \"span\" not in subject\n    assert 'background-color' not in subject\n\n\n    # The email should have two bodies (multipart/alternative with text/plain and text/html)\n    assert msg.is_multipart()\n    assert msg.get_content_type() == 'multipart/alternative'\n\n    # Get the parts\n    parts = list(msg.iter_parts())\n    assert len(parts) == 2\n\n    # First part should be text/plain (the auto-generated plaintext version)\n    text_part = parts[0]\n    assert text_part.get_content_type() == 'text/plain'\n    text_content = text_part.get_content()\n    assert 'So let\\'s see what happens.\\r\\n' in text_content  # The plaintext part\n    assert '(added)' not in text_content # Because apprise only dumb converts the html to text\n\n    # Second part should be text/html with color styling\n    html_part = parts[1]\n    assert html_part.get_content_type() == 'text/html'\n    html_content = html_part.get_content()\n    assert HTML_CHANGED_STYLE or HTML_REMOVED_STYLE in html_content\n    assert HTML_ADDED_STYLE in html_content\n    assert '<' not in html_content\n\n    assert 'some text
    ' in html_content\n    delete_all_watches(client)\n\ndef test_check_notification_markdown_format(client, live_server, measure_memory_usage, datastore_path):\n    set_original_response(datastore_path=datastore_path)\n\n\n    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'\n\n    #####################\n    # Set this up for when we remove the notification from the watch, it should fallback with these details\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\"application-notification_urls\": notification_url,\n              \"application-notification_title\": \"fallback-title  diff_added_lines_test : '{{ diff_added.splitlines()[0] if diff_added else 'diff added didnt split' }}' \" + default_notification_title,\n              \"application-notification_body\": \"*header*\\n\\nsome text\\n\" + default_notification_body,\n              \"application-notification_format\": 'markdown',\n              \"requests-time_between_check-minutes\": 180,\n              'application-fetch_backend': \"html_requests\"},\n        follow_redirects=True\n    )\n\n    assert b\"Settings updated.\" in res.data\n\n    # Add a watch and trigger a HTTP POST\n    test_url = url_for('test_endpoint', _external=True)\n    res = client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": test_url, \"tags\": 'nice one'},\n        follow_redirects=True\n    )\n\n    assert b\"Watch added\" in res.data\n\n    wait_for_all_checks(client)\n    set_longer_modified_response(datastore_path=datastore_path)\n    time.sleep(2)\n\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    time.sleep(3)\n\n    msg_raw = get_last_message_from_smtp_server()\n    assert len(msg_raw) >= 1\n\n    # Parse the email properly using Python's email library\n    msg = message_from_string(msg_raw, policy=email_policy)\n\n    # The email should have two bodies (multipart/alternative with text/plain and text/html)\n    assert msg.is_multipart()\n    assert msg.get_content_type() == 'multipart/alternative'\n    subject = msg['subject']\n    # Subject should always be plaintext and never marked up to anything else\n    assert REMOVED_PLACEMARKER_OPEN not in subject\n    assert CHANGED_PLACEMARKER_OPEN not in subject\n    assert ADDED_PLACEMARKER_OPEN not in subject\n    assert 'diff added didnt split' not in subject\n    assert '(changed) Which is across' in subject\n\n\n    # Get the parts\n    parts = list(msg.iter_parts())\n    assert len(parts) == 2\n\n    # First part should be text/plain (the auto-generated plaintext version)\n    text_part = parts[0]\n    assert text_part.get_content_type() == 'text/plain'\n    text_content = text_part.get_content()\n    # We wont see anything in the \"FALLBACK\" text but that's OK (no added/strikethrough etc)\n    assert 'So let\\'s see what happens.\\r\\n' in text_content  # The plaintext part\n\n\n    # Second part should be text/html and roughly converted from markdown to HTML\n    html_part = parts[1]\n    assert html_part.get_content_type() == 'text/html'\n    html_content = html_part.get_content()\n    assert '
    header
    ' in html_content\n    assert 'So let\\'s see what happens.
    ' in html_content # Additions are  in markdown\n    # the '
    ' will come from apprises conversion, not from our code, we would rather use '
    ' correctly\n    # the '
    ' is actually a nice way to know if apprise done the conversion.\n\n    delete_all_watches(client)\n\n# Custom notification body with HTML, that is either sent as HTML or rendered to plaintext and sent\ndef test_check_notification_email_formats_default_Text_override_HTML(client, live_server, measure_memory_usage, datastore_path):\n\n    # HTML problems? see this\n    # https://github.com/caronc/apprise/issues/633\n\n    set_original_response(datastore_path=datastore_path)\n\n    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'\n    notification_body = f\"\"\"\n\n\n    My Webpage\n\n\n    
    Test
    \n    {default_notification_body}\n\n\n\"\"\"\n\n    #####################\n    # Set this up for when we remove the notification from the watch, it should fallback with these details\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\"application-notification_urls\": notification_url,\n              \"application-notification_title\": \"fallback-title \" + default_notification_title,\n              \"application-notification_body\": notification_body,\n              \"application-notification_format\": 'text',\n              \"requests-time_between_check-minutes\": 180,\n              'application-fetch_backend': \"html_requests\"},\n        follow_redirects=True\n    )\n    assert b\"Settings updated.\" in res.data\n\n    # Add a watch and trigger a HTTP POST\n    test_url = url_for('test_endpoint',content_type=\"text/html\", _external=True)\n    res = client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": test_url, \"tags\": 'nice one'},\n        follow_redirects=True\n    )\n\n    assert b\"Watch added\" in res.data\n\n    #################################### FIRST SITUATION, PLAIN TEXT NOTIFICATION IS WANTED BUT WE HAVE HTML IN OUR TEMPLATE AND CONTENT ##########\n    wait_for_all_checks(client)\n    set_longer_modified_response(datastore_path=datastore_path)\n    time.sleep(2)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    time.sleep(3)\n    msg_raw = get_last_message_from_smtp_server()\n    assert len(msg_raw) >= 1\n    #    with open('/tmp/m.txt', 'w') as f:\n    #        f.write(msg_raw)\n\n    # Parse the email properly using Python's email library\n    msg = message_from_string(msg_raw, policy=email_policy)\n\n    # The email should not have two bodies, should be TEXT only\n    assert not msg.is_multipart()\n    assert msg.get_content_type() == 'text/plain'\n\n    # Get the plain text content\n    text_content = msg.get_content()\n    assert '(added) So let\\'s see what happens.\\r\\n' in text_content  # The plaintext part\n    assert '' in text_content # even tho they added html, they selected plaintext so it should have not got converted\n\n\n    #################################### SECOND SITUATION, HTML IS CORRECTLY PASSED THROUGH TO THE EMAIL ####################\n    set_original_response(datastore_path=datastore_path)\n\n    # Now override as HTML format\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={\n            \"url\": test_url,\n            \"notification_format\": 'html',\n            'fetch_backend': \"html_requests\",\n            \"time_between_check_use_default\": \"y\"},\n        follow_redirects=True\n    )\n    assert b\"Updated watch.\" in res.data\n    wait_for_all_checks(client)\n\n    time.sleep(3)\n    msg_raw = get_last_message_from_smtp_server()\n    assert len(msg_raw) >= 1\n\n    # Parse the email properly using Python's email library\n    msg = message_from_string(msg_raw, policy=email_policy)\n\n    # The email should have two bodies (multipart/alternative)\n    assert msg.is_multipart()\n    assert msg.get_content_type() == 'multipart/alternative'\n\n    # Get the parts\n    parts = list(msg.iter_parts())\n    assert len(parts) == 2\n\n    # First part should be text/plain\n    text_part = parts[0]\n    assert text_part.get_content_type() == 'text/plain'\n    text_content = text_part.get_content()\n    assert '(removed) So let\\'s see what happens.\\r\\n' in text_content  # The plaintext part\n\n    # Second part should be text/html\n    html_part = parts[1]\n    assert html_part.get_content_type() == 'text/html'\n    html_content = html_part.get_content()\n    assert '(removed) So let\\'s see what happens.' in html_content  # the html part\n    assert '<!DOCTYPE html' not in html_content\n    assert 'Test' in html_content\n    assert '<' not in html_content\n\n    delete_all_watches(client)\n\ndef test_check_plaintext_document_plaintext_notification_smtp(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"When following a plaintext document, notification in Plain Text format is sent correctly\"\"\"\n    import os\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(\"Some nice plain text\\nwhich we add some extra data\\nover here\\n\")\n\n    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'\n    notification_body = f\"\"\"{default_notification_body}\"\"\"\n\n    #####################\n    # Set this up for when we remove the notification from the watch, it should fallback with these details\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\"application-notification_urls\": notification_url,\n              \"application-notification_title\": \"fallback-title \" + default_notification_title,\n              \"application-notification_body\": f\"{notification_body}\\nMore output test\\n{ALL_MARKUP_TOKENS}\",\n              \"application-notification_format\": 'text',\n              \"requests-time_between_check-minutes\": 180,\n              'application-fetch_backend': \"html_requests\"},\n        follow_redirects=True\n    )\n    assert b\"Settings updated.\" in res.data\n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', content_type=\"text/plain\", _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    # Change the content\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(\"Some nice plain text\\nwhich we add some extra data\\nAnd let's talk about  tags\\nover here\\n\")\n\n\n    time.sleep(1)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    # Parse the email properly using Python's email library\n    msg = message_from_string(get_last_message_from_smtp_server(), policy=email_policy)\n\n    assert not msg.is_multipart()\n    assert msg.get_content_type() == 'text/plain'\n    body = msg.get_content()\n    # nothing is escaped, raw html stuff in text/plain\n    assert 'talk about <title> tags' in body\n    assert '(added)' in body\n    assert '<br' not in body\n    assert '<' not in body\n    assert '<pre' not in body\n    delete_all_watches(client)\n\ndef test_check_plaintext_document_html_notifications(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"When following a plaintext document, notification in Plain Text format is sent correctly\"\"\"\n    import os\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(\"    Some nice plain text\\nwhich we add some extra data\\nover here\\n\")\n\n    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'\n    notification_body = f\"\"\"{default_notification_body}\"\"\"\n\n    #####################\n    # Set this up for when we remove the notification from the watch, it should fallback with these details\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\"application-notification_urls\": notification_url,\n              \"application-notification_title\": \"fallback-title \" + default_notification_title,\n              \"application-notification_body\": f\"{notification_body}\\nMore output test\\n{ALL_MARKUP_TOKENS}\",\n              \"application-notification_format\": 'html',\n              \"requests-time_between_check-minutes\": 180,\n              'application-fetch_backend': \"html_requests\"},\n        follow_redirects=True\n    )\n    assert b\"Settings updated.\" in res.data\n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', content_type=\"text/plain\", _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    # Change the content\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(\"    Some nice plain text\\nwhich we add some extra data\\nAnd let's talk about <title> tags\\nover here\\n\")\n\n\n    time.sleep(2)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    # Parse the email properly using Python's email library\n    msg = message_from_string(get_last_message_from_smtp_server(), policy=email_policy)\n\n\n    # The email should have two bodies (multipart/alternative)\n    assert msg.is_multipart()\n    assert msg.get_content_type() == 'multipart/alternative'\n\n    # Get the parts\n    parts = list(msg.iter_parts())\n    assert len(parts) == 2\n\n    # First part should be text/plain\n    text_part = parts[0]\n    assert text_part.get_content_type() == 'text/plain'\n    text_content = text_part.get_content()\n    html_part = parts[1]\n    assert html_part.get_content_type() == 'text/html'\n    html_content = html_part.get_content()\n\n\n    assert 'And let\\'s talk about <title> tags\\r\\n' in text_content\n    assert '<br' not in text_content\n    assert '<span' not in text_content\n\n\n    assert 'talk about <title>' not in html_content  # the html part, should have got marked up to < etc\n    assert 'talk about <title>' in html_content\n    # Should be the HTML, but not HTML Color\n    assert 'background-color' not in html_content\n    assert '<br>(added) And let's talk about <title> tags<br>' in html_content\n    assert 'PLACEMARKER' not in html_content\n    assert '<br' not in html_content\n    assert '<pre role=\"article\"' in html_content # Should have got wrapped nicely in email_helpers.py\n\n    # And now for the whitespace retention\n    assert '    Some nice plain text' in html_content\n    assert '(added) And let' in html_content # just to show a single whitespace didnt get touched\n    delete_all_watches(client)\n\n\ndef test_check_plaintext_document_html_color_notifications(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"When following a plaintext document, notification in Plain Text format is sent correctly\"\"\"\n    import os\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(\"Some nice plain text\\nwhich we add some extra data\\nover here\\n\")\n\n    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'\n    notification_body = f\"\"\"{default_notification_body}\"\"\"\n\n    #####################\n    # Set this up for when we remove the notification from the watch, it should fallback with these details\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\"application-notification_urls\": notification_url,\n              \"application-notification_title\": \"fallback-title \" + default_notification_title,\n              \"application-notification_body\": f\"{notification_body}\\nMore output test\\n{ALL_MARKUP_TOKENS}\",\n              \"application-notification_format\": 'htmlcolor',\n              \"requests-time_between_check-minutes\": 180,\n              'application-fetch_backend': \"html_requests\"},\n        follow_redirects=True\n    )\n\n    assert b\"Settings updated.\" in res.data\n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', content_type=\"text/plain\", _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    # Change the content\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(\"Some nice plain text\\nwhich we add some extra data\\nAnd let's talk about <title> tags\\nover here\\n\")\n\n    time.sleep(1)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    # Parse the email properly using Python's email library\n    msg = message_from_string(get_last_message_from_smtp_server(), policy=email_policy)\n\n    # The email should have two bodies (multipart/alternative)\n    assert msg.is_multipart()\n    assert msg.get_content_type() == 'multipart/alternative'\n\n    # Get the parts\n    parts = list(msg.iter_parts())\n    assert len(parts) == 2\n\n    # First part should be text/plain\n    text_part = parts[0]\n    assert text_part.get_content_type() == 'text/plain'\n    text_content = text_part.get_content()\n    html_part = parts[1]\n    assert html_part.get_content_type() == 'text/html'\n    html_content = html_part.get_content()\n\n\n    assert 'And let\\'s talk about <title> tags\\r\\n' in text_content\n    assert '<br' not in text_content\n    assert '<span' not in text_content\n\n    assert 'talk about <title>' not in html_content  # the html part, should have got marked up to < etc\n    assert 'talk about <title>' in html_content\n    # Should be the HTML, but not HTML Color\n    assert 'background-color' in html_content\n    assert '(added) And let' not in html_content\n    assert '<br' not in html_content\n    assert '<br>' in html_content\n    assert '<pre role=\"article\"' in html_content # Should have got wrapped nicely in email_helpers.py\n    delete_all_watches(client)\n\ndef test_check_html_document_plaintext_notification(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"When following a HTML document, notification in Plain Text format is sent correctly\"\"\"\n    import os\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(\"<html><body>some stuff<br>and more stuff<br>and even more stuff<br></body></html>\")\n\n    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com'\n    notification_body = f\"\"\"{default_notification_body}\"\"\"\n\n    #####################\n    # Set this up for when we remove the notification from the watch, it should fallback with these details\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\"application-notification_urls\": notification_url,\n              \"application-notification_title\": \"fallback-title \" + default_notification_title,\n              \"application-notification_body\": f\"{notification_body}\\nMore output test\\n{ALL_MARKUP_TOKENS}\",\n              \"application-notification_format\": 'text',\n              \"requests-time_between_check-minutes\": 180,\n              'application-fetch_backend': \"html_requests\"},\n        follow_redirects=True\n    )\n\n    assert b\"Settings updated.\" in res.data\n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', content_type=\"text/html\", _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(\"<html><body>sxome stuff<br>and more stuff<br>lets slip this in<br>and this in<br>and even more stuff<br><tag></body></html>\")\n\n    time.sleep(0.1)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n\n    # Parse the email properly using Python's email library\n    msg = message_from_string(get_last_message_from_smtp_server(), policy=email_policy)\n\n    assert not msg.is_multipart()\n    assert msg.get_content_type() == 'text/plain'\n    body = msg.get_content()\n    assert '<tag>' in body # Should have got converted from original HTML to plaintext\n    assert '(changed) some stuff\\r\\n' in body\n    assert 'PLACEMARKER' not in body\n    assert '(into) sxome stuff\\r\\n' in body\n    assert '(added) lets slip this in\\r\\n' in body\n    assert '(added) and this in\\r\\n' in body\n    assert ' ' not in body\n\n\n    delete_all_watches(client)\n\n\ndef test_check_html_notification_with_apprise_format_is_html(client, live_server, measure_memory_usage, datastore_path):\n    ##  live_server_setup(live_server) # Setup on conftest per function\n    set_original_response(datastore_path=datastore_path)\n\n\n    notification_url = f'mailto://changedetection@{smtp_test_server}:11025/?to=fff@home.com&format=html'\n\n    #####################\n    # Set this up for when we remove the notification from the watch, it should fallback with these details\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\"application-notification_urls\": notification_url,\n              \"application-notification_title\": \"fallback-title \" + default_notification_title,\n              \"application-notification_body\": \"some text\\nfallback-body<br> \" + default_notification_body,\n              \"application-notification_format\": 'html',\n              \"requests-time_between_check-minutes\": 180,\n              'application-fetch_backend': \"html_requests\"},\n        follow_redirects=True\n    )\n    assert b\"Settings updated.\" in res.data\n\n    # Add a watch and trigger a HTTP POST\n    test_url = url_for('test_endpoint', _external=True)\n    res = client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": test_url, \"tags\": 'nice one'},\n        follow_redirects=True\n    )\n\n    assert b\"Watch added\" in res.data\n\n    wait_for_all_checks(client)\n    set_longer_modified_response(datastore_path=datastore_path)\n    time.sleep(2)\n\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    time.sleep(3)\n\n    msg_raw = get_last_message_from_smtp_server()\n    assert len(msg_raw) >= 1\n\n    # Parse the email properly using Python's email library\n    msg = message_from_string(msg_raw, policy=email_policy)\n\n    # The email should have two bodies (multipart/alternative with text/plain and text/html)\n    assert msg.is_multipart()\n    assert msg.get_content_type() == 'multipart/alternative'\n\n    # Get the parts\n    parts = list(msg.iter_parts())\n    assert len(parts) == 2\n\n    # First part should be text/plain (the auto-generated plaintext version)\n    text_part = parts[0]\n    assert text_part.get_content_type() == 'text/plain'\n    text_content = text_part.get_content()\n    assert '(added) So let\\'s see what happens.\\r\\n' in text_content  # The plaintext part\n    assert 'fallback-body\\r\\n' in text_content  # The plaintext part\n\n    # Second part should be text/html\n    html_part = parts[1]\n    assert html_part.get_content_type() == 'text/html'\n    html_content = html_part.get_content()\n    assert 'some text<br>' in html_content  # We converted \\n from the notification body\n    assert 'fallback-body<br>' in html_content  # kept the original <br>\n    assert '(added) So let\\'s see what happens.<br>' in html_content  # the html part\n    delete_all_watches(client)"
  },
  {
    "path": "changedetectionio/tests/test_access_control.py",
    "content": "from .util import live_server_setup, wait_for_all_checks\nfrom flask import url_for\nimport time\n\ndef test_check_access_control(app, client, live_server, measure_memory_usage, datastore_path):\n    # Still doesnt work, but this is closer.\n   #  live_server_setup(live_server) # Setup on conftest per function\n\n    with app.test_client(use_cookies=True) as c:\n        # Check we don't have any password protection enabled yet.\n        res = c.get(url_for(\"settings.settings_page\"))\n        assert b\"Remove password\" not in res.data\n\n        # add something that we can hit via diff page later\n        res = c.post(\n            url_for(\"imports.import_page\"),\n            data={\"urls\": url_for('test_random_content_endpoint', _external=True)},\n            follow_redirects=True\n        )\n\n        assert b\"1 Imported\" in res.data\n        # causes a 'Popped wrong request context.' error when client. is accessed?\n        wait_for_all_checks(client)\n\n        res = c.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n        assert b'Queued 1 watch for rechecking.' in res.data\n        wait_for_all_checks(client)\n\n\n        # Enable password check and diff page access bypass\n        res = c.post(\n            url_for(\"settings.settings_page\"),\n            data={\"application-password\": \"foobar\",\n                  \"application-shared_diff_access\": \"True\",\n                  \"requests-time_between_check-minutes\": 180,\n                  'application-fetch_backend': \"html_requests\"},\n            follow_redirects=True\n        )\n\n        assert b\"Password protection enabled.\" in res.data\n\n        # Check we hit the login\n        res = c.get(url_for(\"watchlist.index\"), follow_redirects=True)\n        # Should be logged out\n        assert b\"Login\" in res.data\n\n        # The diff page should return something valid when logged out\n        res = c.get(url_for(\"ui.ui_diff.diff_history_page\", uuid=\"first\"))\n        assert b'Random content' in res.data\n\n        # access to assets should work (check_authentication)\n        res = c.get(url_for('static_content', group='js', filename='jquery-3.6.0.min.js'))\n        assert res.status_code == 200\n        res = c.get(url_for('static_content', group='styles', filename='styles.css'))\n        assert res.status_code == 200\n        res = c.get(url_for('static_content', group='styles', filename='404-testetest.css'))\n        assert res.status_code == 404\n\n        # Access to screenshots should be limited by 'shared_diff_access'\n        path = url_for('static_content', group='screenshot', filename='random-uuid-that-will-404.png', _external=True)\n        res = c.get(path)\n        assert res.status_code == 404\n\n        # Check wrong password does not let us in\n        res = c.post(\n            url_for(\"login\"),\n            data={\"password\": \"WRONG PASSWORD\"},\n            follow_redirects=True\n        )\n\n        assert b\"LOG OUT\" not in res.data\n        assert b\"Incorrect password\" in res.data\n\n\n        # Menu should not be available yet\n        #        assert b\"SETTINGS\" not in res.data\n        #        assert b\"BACKUP\" not in res.data\n        #        assert b\"IMPORT\" not in res.data\n\n        # defaultuser@changedetection.io is actually hardcoded for now, we only use a single password\n        res = c.post(\n            url_for(\"login\"),\n            data={\"password\": \"foobar\"},\n            follow_redirects=True\n        )\n\n        # Yes we are correctly logged in\n        assert b\"LOG OUT\" in res.data\n\n        # 598 - Password should be set and not accidently removed\n        res = c.post(\n            url_for(\"settings.settings_page\"),\n            data={\n                  \"requests-time_between_check-minutes\": 180,\n                  'application-fetch_backend': \"html_requests\"},\n            follow_redirects=True\n        )\n\n        res = c.get(url_for(\"logout\"),\n            follow_redirects=True)\n\n        assert b\"Login\" in res.data\n\n        res = c.get(url_for(\"settings.settings_page\"),\n            follow_redirects=True)\n\n\n        assert b\"Login\" in res.data\n\n        res = c.get(url_for(\"login\"))\n        assert b\"Login\" in res.data\n\n\n        res = c.post(\n            url_for(\"login\"),\n            data={\"password\": \"foobar\"},\n            follow_redirects=True\n        )\n\n        # Yes we are correctly logged in\n        assert b\"LOG OUT\" in res.data\n\n        res = c.get(url_for(\"settings.settings_page\"))\n\n        # Menu should be available now\n        assert b\"SETTINGS\" in res.data\n        assert b\"IMPORT\" in res.data\n        assert b\"LOG OUT\" in res.data\n        assert b\"time_between_check-minutes\" in res.data\n        assert b\"fetch_backend\" in res.data\n\n        ##################################################\n        # Remove password button, and check that it worked\n        ##################################################\n        res = c.post(\n            url_for(\"settings.settings_page\"),\n            data={\n                \"requests-time_between_check-minutes\": 180,\n                \"application-fetch_backend\": \"html_webdriver\",\n                \"application-removepassword_button\": \"Remove password\"\n            },\n            follow_redirects=True,\n        )\n        assert b\"Password protection removed.\" in res.data\n        assert b\"LOG OUT\" not in res.data\n\n        ############################################################\n        # Be sure a blank password doesnt setup password protection\n        ############################################################\n        res = c.post(\n            url_for(\"settings.settings_page\"),\n            data={\"application-password\": \"\",\n                  \"requests-time_between_check-minutes\": 180,\n                  'application-fetch_backend': \"html_requests\"},\n            follow_redirects=True\n        )\n\n        assert b\"Password protection enabled\" not in res.data\n\n        # Now checking the diff access\n        # Enable password check and diff page access bypass\n        res = c.post(\n            url_for(\"settings.settings_page\"),\n            data={\"application-password\": \"foobar\",\n                  # Should be disabled\n                  \"application-shared_diff_access\": \"\",\n                  \"requests-time_between_check-minutes\": 180,\n                  'application-fetch_backend': \"html_requests\"},\n            follow_redirects=True\n        )\n\n        assert b\"Password protection enabled.\" in res.data\n\n        # Check we hit the login\n        res = c.get(url_for(\"watchlist.index\"), follow_redirects=True)\n        # Should be logged out\n        assert b\"Login\" in res.data\n\n        # Access to screenshots should be limited by 'shared_diff_access'\n        res = c.get(url_for('static_content', group='screenshot', filename='random-uuid-that-will-403.png'))\n        assert res.status_code == 403\n\n        # The diff page should return something valid when logged out\n        res = c.get(url_for(\"ui.ui_diff.diff_history_page\", uuid=\"first\"))\n        assert b'Random content' not in res.data\n"
  },
  {
    "path": "changedetectionio/tests/test_add_replace_remove_filter.py",
    "content": "#!/usr/bin/env python3\n\nimport os.path\nimport os\n\nfrom flask import url_for\nfrom .util import live_server_setup, wait_for_all_checks, wait_for_notification_endpoint_output, delete_all_watches\nimport time\n\nfrom ..diff import ADDED_PLACEMARKER_OPEN\n\n\ndef set_original(datastore_path, excluding=None, add_line=None):\n    test_return_data = \"\"\"<html>\n     <body>\n     <p>Some initial text</p>\n     <p>So let's see what happens.</p>\n     <p>and a new line!</p>\n     <p>The golden line</p>\n     <p>A BREAK TO MAKE THE TOP LINE STAY AS \"REMOVED\" OR IT WILL GET COUNTED AS \"CHANGED INTO\"</p>\n     <p>Something irrelevant</p>          \n     </body>\n     </html>\n    \"\"\"\n\n    if add_line:\n        c=test_return_data.splitlines()\n        c.insert(5, add_line)\n        test_return_data = \"\\n\".join(c)\n\n    if excluding:\n        output = \"\"\n        for i in test_return_data.splitlines():\n            if not excluding in i:\n                output += f\"{i}\\n\"\n\n        test_return_data = output\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n\n# def test_setup(client, live_server, measure_memory_usage, datastore_path):\n   #  live_server_setup(live_server) # Setup on conftest per function\n\ndef test_check_removed_line_contains_trigger(client, live_server, measure_memory_usage, datastore_path):\n\n    # Give the endpoint time to spin up\n    set_original(datastore_path=datastore_path)\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    # Give the thread time to pick it up\n    wait_for_all_checks(client)\n\n    # Goto the edit page, add our ignore text\n    # Add our URL to the import page\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={\"trigger_text\": 'The golden line',\n              \"url\": test_url,\n              'fetch_backend': \"html_requests\",\n              'filter_text_removed': 'y',\n              \"time_between_check_use_default\": \"y\"},\n        follow_redirects=True\n    )\n\n    assert b\"Updated watch.\" in res.data\n    wait_for_all_checks(client)\n    set_original(excluding='Something irrelevant', datastore_path=datastore_path)\n\n    # A line thats not the trigger should not trigger anything\n    res = client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    assert b'Queued 1 watch for rechecking.' in res.data\n    wait_for_all_checks(client)\n    time.sleep(0.5)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'has-unread-changes' not in res.data\n\n    # The trigger line is REMOVED,  this should trigger\n    set_original(excluding='The golden line', datastore_path=datastore_path)\n\n    # Check in the processor here what's going on, its triggering empty-reply and no change.\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'has-unread-changes' in res.data\n\n    time.sleep(1)\n\n    # Now add it back, and we should not get a trigger\n    client.get(url_for(\"ui.mark_all_viewed\"), follow_redirects=True)\n    time.sleep(0.2)\n\n    time.sleep(1)\n    set_original(excluding=None, datastore_path=datastore_path)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n    time.sleep(1)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'has-unread-changes' not in res.data\n\n    # Remove it again, and we should get a trigger\n    set_original(excluding='The golden line', datastore_path=datastore_path)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'has-unread-changes' in res.data\n\n    delete_all_watches(client)\n\n\ndef test_check_add_line_contains_trigger(client, live_server, measure_memory_usage, datastore_path):\n    \n    delete_all_watches(client)\n    time.sleep(1)\n\n    # Give the endpoint time to spin up\n    test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://') + \"?xxx={{ watch_url }}\"\n\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\"application-notification_title\": \"New ChangeDetection.io Notification - {{ watch_url }}\",\n              # triggered_text will contain multiple lines\n              \"application-notification_body\": 'triggered text was -{{triggered_text}}- ### 网站监测 内容更新了 ####',\n              # https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation\n              \"application-notification_urls\": test_notification_url,\n              \"application-notification_format\": 'text',\n              \"application-minutes_between_check\": 180,\n              \"application-fetch_backend\": \"html_requests\"\n              },\n        follow_redirects=True\n    )\n    assert b'Settings updated' in res.data\n\n    set_original(datastore_path=datastore_path)\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    # Give the thread time to pick it up\n    wait_for_all_checks(client)\n    # Goto the edit page, add our ignore text\n    # Add our URL to the import page\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={\"trigger_text\": 'Oh yes please',\n              \"url\": test_url,\n              'processor': 'text_json_diff',\n              'fetch_backend': \"html_requests\",\n              'filter_text_removed': '',\n              'filter_text_added': 'y',\n              \"time_between_check_use_default\": \"y\"},\n        follow_redirects=True\n    )\n    assert b\"Updated watch.\" in res.data\n    wait_for_all_checks(client)\n    set_original(excluding='Something irrelevant', datastore_path=datastore_path)\n\n    # A line thats not the trigger should not trigger anything\n    res = client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    assert b'Queued 1 watch for rechecking.' in res.data\n\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'has-unread-changes' not in res.data\n\n    # The trigger line is ADDED,  this should trigger\n    set_original(add_line='<p>Oh yes please</p>', datastore_path=datastore_path)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"watchlist.index\"))\n\n    assert b'has-unread-changes' in res.data\n\n    # Takes a moment for apprise to fire\n    wait_for_notification_endpoint_output(datastore_path=datastore_path)\n    assert os.path.isfile(os.path.join(datastore_path, \"notification.txt\")), \"Notification fired because I can see the output file\"\n    with open(os.path.join(datastore_path, \"notification.txt\"), 'rb') as f:\n        response = f.read()\n        assert ADDED_PLACEMARKER_OPEN.encode('utf-8') not in response #  _apply_diff_filtering shouldnt add something here\n        assert b'-Oh yes please' in response\n        assert '网站监测 内容更新了'.encode('utf-8') in response\n\n    delete_all_watches(client)\n"
  },
  {
    "path": "changedetectionio/tests/test_api.py",
    "content": "#!/usr/bin/env python3\n\nimport time\nfrom flask import url_for\nfrom .util import live_server_setup, wait_for_all_checks, delete_all_watches\nimport os\n\nimport json\nimport uuid\n\n\ndef set_original_response(datastore_path):\n    test_return_data = \"\"\"<html>\n       <body>\n     Some initial text<br>\n     <p>Which is across multiple lines</p>\n     <br>\n     So let's see what happens.  <br>\n     <div id=\"sametext\">Some text thats the same</div>\n     <div id=\"changetext\">Some text that will change</div>\n     </body>\n     </html>\n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n    return None\n\n\ndef set_modified_response(datastore_path):\n    test_return_data = \"\"\"<html>\n       <body>\n     Some initial text<br>\n     <p>which has this one new line</p>\n     <br>\n     So let's see what happens.  <br>\n     <div id=\"sametext\">Some text thats the same</div>\n     <div id=\"changetext\">Some text that changes</div>\n     </body>\n     </html>\n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n\n    return None\n\ndef is_valid_uuid(val):\n    try:\n        uuid.UUID(str(val))\n        return True\n    except ValueError:\n        return False\n\n\n# def test_setup(client, live_server, measure_memory_usage, datastore_path):\n   #  live_server_setup(live_server) # Setup on conftest per function\n\n\ndef test_api_simple(client, live_server, measure_memory_usage, datastore_path):\n\n\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n\n    # Create a watch\n    set_original_response(datastore_path=datastore_path)\n\n    # Validate bad URL\n    test_url = url_for('test_endpoint', _external=True )\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\"url\": \"h://xxxxxxxxxom\"}),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n        follow_redirects=True\n    )\n    assert res.status_code == 400\n\n    # Create new\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\"url\": test_url, 'tag': \"One, Two\", \"title\": \"My test URL\"}),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n        follow_redirects=True\n    )\n\n    assert is_valid_uuid(res.json.get('uuid'))\n    watch_uuid = res.json.get('uuid')\n    assert res.status_code == 201\n\n    wait_for_all_checks(client)\n\n    # Verify its in the list and that recheck worked\n    res = client.get(\n        url_for(\"createwatch\", tag=\"OnE\"),\n        headers={'x-api-key': api_key}\n    )\n    assert watch_uuid in res.json.keys()\n    before_recheck_info = res.json[watch_uuid]\n\n    assert before_recheck_info['last_checked'] != 0\n\n    #705 `last_changed` should be zero on the first check\n    assert before_recheck_info['last_changed'] == 0\n    assert before_recheck_info['title'] == 'My test URL'\n\n    # Check the limit by tag doesnt return anything when nothing found\n    res = client.get(\n        url_for(\"createwatch\", tag=\"Something else entirely\"),\n        headers={'x-api-key': api_key}\n    )\n    assert len(res.json) == 0\n    time.sleep(2)\n    wait_for_all_checks(client)\n    set_modified_response(datastore_path=datastore_path)\n    # Trigger recheck of all ?recheck_all=1\n    res = client.get(\n        url_for(\"createwatch\", recheck_all='1'),\n        headers={'x-api-key': api_key},\n    )\n    wait_for_all_checks(client)\n\n    time.sleep(2)\n    # Did the recheck fire?\n    res = client.get(\n        url_for(\"createwatch\"),\n        headers={'x-api-key': api_key},\n    )\n    after_recheck_info = res.json[watch_uuid]\n    assert after_recheck_info['last_checked'] != before_recheck_info['last_checked']\n    assert after_recheck_info['last_changed'] != 0\n\n    # #2877 When run in a slow fetcher like playwright etc\n    assert after_recheck_info['last_changed'] ==  after_recheck_info['last_checked']\n\n    # Check history index list\n    res = client.get(\n        url_for(\"watchhistory\", uuid=watch_uuid),\n        headers={'x-api-key': api_key},\n    )\n    watch_history = res.json\n    assert len(res.json) == 2, \"Should have two history entries (the original and the changed)\"\n\n    # Fetch a snapshot by timestamp, check the right one was found\n    res = client.get(\n        url_for(\"watchsinglehistory\", uuid=watch_uuid, timestamp=list(res.json.keys())[-1]),\n        headers={'x-api-key': api_key},\n    )\n    assert b'which has this one new line' in res.data\n\n    # Fetch a snapshot by 'latest'', check the right one was found\n    res = client.get(\n        url_for(\"watchsinglehistory\", uuid=watch_uuid, timestamp='latest'),\n        headers={'x-api-key': api_key},\n    )\n    assert b'which has this one new line' in res.data\n    assert b'<div id' not in res.data\n\n    # Fetch the HTML of the latest one\n    res = client.get(\n        url_for(\"watchsinglehistory\", uuid=watch_uuid, timestamp='latest')+\"?html=1\",\n        headers={'x-api-key': api_key},\n    )\n    assert b'which has this one new line' in res.data\n    assert b'<div id' in res.data\n\n\n    # Fetch the difference between two versions (default text format)\n    res = client.get(\n        url_for(\"watchhistorydiff\", uuid=watch_uuid, from_timestamp='previous', to_timestamp='latest'),\n        headers={'x-api-key': api_key},\n    )\n    assert b'(changed) Which is across' in res.data\n    assert b'Some text thats the same' in res.data\n\n    # Fetch the difference between two versions (default text format)\n    res = client.get(\n        url_for(\"watchhistorydiff\", uuid=watch_uuid, from_timestamp='previous', to_timestamp='latest')+\"?changesOnly=true\",\n        headers={'x-api-key': api_key},\n    )\n    assert b'Some text thats the same' not in res.data\n\n    # Test htmlcolor format\n    res = client.get(\n        url_for(\"watchhistorydiff\", uuid=watch_uuid, from_timestamp='previous', to_timestamp='latest')+'?format=htmlcolor',\n        headers={'x-api-key': api_key},\n    )\n    assert b'aria-label=\"Changed text\" title=\"Changed text\">Which is across multiple lines' in res.data\n\n    # Test html format\n    res = client.get(\n        url_for(\"watchhistorydiff\", uuid=watch_uuid, from_timestamp='previous', to_timestamp='latest')+'?format=html',\n        headers={'x-api-key': api_key},\n    )\n    assert res.status_code == 200\n    assert b'<br>' in res.data\n\n    # Test markdown format\n    res = client.get(\n        url_for(\"watchhistorydiff\", uuid=watch_uuid, from_timestamp='previous', to_timestamp='latest')+'?format=markdown',\n        headers={'x-api-key': api_key},\n    )\n    assert res.status_code == 200\n\n    # Test new diff preference parameters\n    # Test removed=false (should hide removed content)\n    res = client.get(\n        url_for(\"watchhistorydiff\", uuid=watch_uuid, from_timestamp='previous', to_timestamp='latest')+'?removed=false',\n        headers={'x-api-key': api_key},\n    )\n    # Should not contain removed content indicator\n    assert b'(removed)' not in res.data\n    # Should still contain added content\n    assert b'(added)' in res.data or b'which has this one new line' in res.data\n\n    # Test added=false (should hide added content)\n    # Note: The test data has replacements, not pure additions, so we test differently\n    res = client.get(\n        url_for(\"watchhistorydiff\", uuid=watch_uuid, from_timestamp='previous', to_timestamp='latest')+'?added=false&replaced=false',\n        headers={'x-api-key': api_key},\n    )\n    # With both added and replaced disabled, should have minimal content\n    # Should not contain added indicators\n    assert b'(added)' not in res.data\n\n    # Test replaced=false (should hide replaced/changed content)\n    res = client.get(\n        url_for(\"watchhistorydiff\", uuid=watch_uuid, from_timestamp='previous', to_timestamp='latest')+'?replaced=false',\n        headers={'x-api-key': api_key},\n    )\n    # Should not contain changed content indicator\n    assert b'(changed)' not in res.data\n\n    # Test type=diffWords for word-level diff\n    res = client.get(\n        url_for(\"watchhistorydiff\", uuid=watch_uuid, from_timestamp='previous', to_timestamp='latest')+'?type=diffWords&format=htmlcolor',\n        headers={'x-api-key': api_key},\n    )\n    # Should contain HTML formatted diff\n    assert res.status_code == 200\n    assert len(res.data) > 0\n\n    # Test combined parameters: show only additions with word diff\n    res = client.get(\n        url_for(\"watchhistorydiff\", uuid=watch_uuid, from_timestamp='previous', to_timestamp='latest')+'?removed=false&replaced=false&type=diffWords',\n        headers={'x-api-key': api_key},\n    )\n    assert res.status_code == 200\n    # Should not contain removed or changed markers\n    assert b'(removed)' not in res.data\n    assert b'(changed)' not in res.data\n\n\n    # Fetch the whole watch\n    res = client.get(\n        url_for(\"watch\", uuid=watch_uuid),\n        headers={'x-api-key': api_key}\n    )\n    watch = res.json\n    # @todo how to handle None/default global values?\n    assert watch['history_n'] == 2, \"Found replacement history section, which is in its own API\"\n\n    assert watch.get('viewed') == False\n    # Loading the most recent snapshot should force viewed to become true\n    client.get(url_for(\"ui.ui_diff.diff_history_page\", uuid=\"first\"), follow_redirects=True)\n\n    time.sleep(3)\n    # Fetch the whole watch again, viewed should be true\n    res = client.get(\n        url_for(\"watch\", uuid=watch_uuid),\n        headers={'x-api-key': api_key}\n    )\n    watch = res.json\n    assert watch.get('viewed') == True\n\n    # basic systeminfo check\n    res = client.get(\n        url_for(\"systeminfo\"),\n        headers={'x-api-key': api_key},\n    )\n    assert res.json.get('watch_count') == 1\n    assert res.json.get('uptime') > 0.5\n\n    ######################################################\n    # Mute and Pause, check it worked\n    res = client.get(\n        url_for(\"watch\", uuid=watch_uuid, paused='paused'),\n        headers={'x-api-key': api_key}\n    )\n    assert b'OK' in res.data\n    res = client.get(\n        url_for(\"watch\", uuid=watch_uuid,  muted='muted'),\n        headers={'x-api-key': api_key}\n    )\n    assert b'OK' in res.data\n    res = client.get(\n        url_for(\"watch\", uuid=watch_uuid),\n        headers={'x-api-key': api_key}\n    )\n    assert res.json.get('paused') == True\n    assert res.json.get('notification_muted') == True\n\n    # Now unpause, unmute\n    res = client.get(\n        url_for(\"watch\", uuid=watch_uuid,  muted='unmuted'),\n        headers={'x-api-key': api_key}\n    )\n    assert b'OK' in res.data\n    res = client.get(\n        url_for(\"watch\", uuid=watch_uuid, paused='unpaused'),\n        headers={'x-api-key': api_key}\n    )\n    assert b'OK' in res.data\n    res = client.get(\n        url_for(\"watch\", uuid=watch_uuid),\n        headers={'x-api-key': api_key}\n    )\n    assert res.json.get('paused') == 0\n    assert res.json.get('notification_muted') == 0\n    ######################################################\n\n\n\n\n\n    # Finally delete the watch\n    res = client.delete(\n        url_for(\"watch\", uuid=watch_uuid),\n        headers={'x-api-key': api_key},\n    )\n    assert res.status_code == 204\n\n    # Check via a relist\n    res = client.get(\n        url_for(\"createwatch\"),\n        headers={'x-api-key': api_key}\n    )\n    assert len(res.json) == 0, \"Watch list should be empty\"\n\ndef test_roundtrip_API(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test the full round trip, this way we test the default Model fits back into OpenAPI spec\n    :param client:\n    :param live_server:\n    :param measure_memory_usage:\n    :param datastore_path:\n    :return:\n    \"\"\"\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n\n    set_original_response(datastore_path=datastore_path)\n    test_url = url_for('test_endpoint', _external=True)\n\n    # Create new\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\"url\": test_url}),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n        follow_redirects=True\n    )\n\n    assert res.status_code == 201\n    uuid = res.json.get('uuid')\n\n    # Now fetch it and send it back\n\n    res = client.get(\n        url_for(\"watch\", uuid=uuid),\n        headers={'x-api-key': api_key}\n    )\n\n    watch=res.json\n\n    # Be sure that 'readOnly' values are never updated in the real watch\n    watch['last_changed'] = 454444444444\n    watch['date_created'] = 454444444444\n\n    # HTTP PUT ( UPDATE an existing watch )\n    res = client.put(\n        url_for(\"watch\", uuid=uuid),\n        headers={'x-api-key': api_key, 'content-type': 'application/json'},\n        data=json.dumps(watch),\n    )\n    if res.status_code != 200:\n        print(f\"\\n=== PUT failed with {res.status_code} ===\")\n        print(f\"Error: {res.data}\")\n    assert res.status_code == 200, \"HTTP PUT update was sent OK\"\n\n    res = client.get(\n        url_for(\"watch\", uuid=uuid),\n        headers={'x-api-key': api_key}\n    )\n    last_changed = res.json.get('last_changed')\n    assert last_changed != 454444444444\n    assert last_changed != \"454444444444\"\n\n    date_created = res.json.get('date_created')\n    assert date_created != 454444444444\n    assert date_created != \"454444444444\"\n\n\ndef test_access_denied(client, live_server, measure_memory_usage, datastore_path):\n    # `config_api_token_enabled` Should be On by default\n    res = client.get(\n        url_for(\"createwatch\")\n    )\n    assert res.status_code == 403\n\n    res = client.get(\n        url_for(\"createwatch\"),\n        headers={'x-api-key': \"something horrible\"}\n    )\n    assert res.status_code == 403\n\n    # Disable config_api_token_enabled and it should work\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\n            \"requests-time_between_check-minutes\": 180,\n            \"application-fetch_backend\": \"html_requests\",\n            \"application-api_access_token_enabled\": \"\"\n        },\n        follow_redirects=True\n    )\n\n    assert b\"Settings updated.\" in res.data\n\n    res = client.get(\n        url_for(\"createwatch\")\n    )\n    assert res.status_code == 200\n\n    # Cleanup everything\n    delete_all_watches(client)\n\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\n            \"requests-time_between_check-minutes\": 180,\n            \"application-fetch_backend\": \"html_requests\",\n            \"application-api_access_token_enabled\": \"y\"\n        },\n        follow_redirects=True\n    )\n    assert b\"Settings updated.\" in res.data\n\ndef test_api_watch_PUT_update(client, live_server, measure_memory_usage, datastore_path):\n\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n    # Create a watch\n    set_original_response(datastore_path=datastore_path)\n    test_url = url_for('test_endpoint', _external=True)\n\n    # Create new\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\"url\": test_url,\n                         'tag': \"One, Two\",\n                         \"title\": \"My test URL\",\n                         'headers': {'cookie': 'yum'},\n                         \"conditions\": [\n                             {\n                                 \"field\": \"page_filtered_text\",\n                                 \"operator\": \"contains_regex\",\n                                 \"value\": \".\"  # contains anything\n                             }\n                         ],\n                         \"conditions_match_logic\": \"ALL\",\n                         }\n                        ),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n        follow_redirects=True\n    )\n\n    if res.status_code != 201:\n        print(f\"\\n=== POST createwatch failed with {res.status_code} ===\")\n        print(f\"Response: {res.data}\")\n    assert res.status_code == 201\n\n    wait_for_all_checks(client)\n    # Get a listing, it will be the first one\n    res = client.get(\n        url_for(\"createwatch\"),\n        headers={'x-api-key': api_key}\n    )\n\n    watch_uuid = list(res.json.keys())[0]\n    assert not res.json[watch_uuid].get('viewed'), 'A newly created watch can only be unviewed'\n\n    # Check in the edit page just to be sure\n    res = client.get(\n        url_for(\"ui.ui_edit.edit_page\", uuid=watch_uuid),\n    )\n    assert b\"cookie: yum\" in res.data, \"'cookie: yum' found in 'headers' section\"\n    assert b\"One\" in res.data, \"Tag 'One' was found\"\n    assert b\"Two\" in res.data, \"Tag 'Two' was found\"\n\n    # HTTP PUT ( UPDATE an existing watch )\n    res = client.put(\n        url_for(\"watch\", uuid=watch_uuid),\n        headers={'x-api-key': api_key, 'content-type': 'application/json'},\n        data=json.dumps({\n            \"title\": \"new title\",\n            'time_between_check': {'minutes': 552},\n            'headers': {'cookie': 'all eaten'},\n            'last_viewed': int(time.time())\n        }),\n    )\n    assert res.status_code == 200, \"HTTP PUT update was sent OK\"\n\n    # HTTP GET single watch, title should be updated\n    res = client.get(\n        url_for(\"watch\", uuid=watch_uuid),\n        headers={'x-api-key': api_key}\n    )\n    assert res.json.get('title') == 'new title'\n    assert res.json.get('viewed'), 'With the timestamp greater than \"changed\" a watch can be updated to viewed'\n\n    # Check in the edit page just to be sure\n    res = client.get(\n        url_for(\"ui.ui_edit.edit_page\", uuid=watch_uuid),\n    )\n    assert b\"new title\" in res.data, \"new title found in edit page\"\n    assert b\"552\" in res.data, \"552 minutes found in edit page\"\n    assert b\"One\" in res.data, \"Tag 'One' was found\"\n    assert b\"Two\" in res.data, \"Tag 'Two' was found\"\n    assert b\"cookie: all eaten\" in res.data, \"'cookie: all eaten' found in 'headers' section\"\n\n    ######################################################\n\n    # HTTP PUT try a field that doesn't exist\n\n    # HTTP PUT an update\n    res = client.put(\n        url_for(\"watch\", uuid=watch_uuid),\n        headers={'x-api-key': api_key, 'content-type': 'application/json'},\n        data=json.dumps({\"title\": \"new title\", \"some other field\": \"uh oh\"}),\n    )\n\n    assert res.status_code == 400, \"Should get error 400 when we give a field that doesnt exist\"\n    # Backend validation now rejects unknown fields with a clear error message\n    assert (b'Unknown field' in res.data or\n            b'Additional properties are not allowed' in res.data or\n            b'Unevaluated properties are not allowed' in res.data or\n            b'does not match any of the regexes' in res.data), \\\n            \"Should reject unknown fields with validation error\"\n\n\n    # Try a XSS URL\n    res = client.put(\n        url_for(\"watch\", uuid=watch_uuid),\n        headers={'x-api-key': api_key, 'content-type': 'application/json'},\n        data=json.dumps({\n            'url': 'javascript:alert(document.domain)'\n        }),\n    )\n    assert res.status_code == 400\n\n    # Cleanup everything\n    delete_all_watches(client)\n\n\ndef test_api_import(client, live_server, measure_memory_usage, datastore_path):\n\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n\n    # Test 1: Basic import with tag\n    res = client.post(\n        url_for(\"import\") + \"?tag=import-test\",\n        data='https://website1.com\\r\\nhttps://website2.com',\n        # We removed  'content-type': 'text/plain', the Import API should assume this if none is set #3547 #3542\n        headers={'x-api-key': api_key},\n        follow_redirects=True\n    )\n\n    assert res.status_code == 200\n    assert len(res.json) == 2\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b\"https://website1.com\" in res.data\n    assert b\"https://website2.com\" in res.data\n\n    # Should see the new tag in the tag/groups list\n    res = client.get(url_for('tags.tags_overview_page'))\n    assert b'import-test' in res.data\n\n    # Test 2: Import with watch configuration fields (issue #3845)\n    # Test string field (include_filters), boolean (paused), and processor\n    import urllib.parse\n    params = urllib.parse.urlencode({\n        'tag': 'config-test',\n        'include_filters': 'div.content',\n        'paused': 'true',\n        'processor': 'text_json_diff',\n        'title': 'Imported with Config'\n    })\n\n    res = client.post(\n        url_for(\"import\") + \"?\" + params,\n        data='https://website3.com',\n        headers={'x-api-key': api_key},\n        follow_redirects=True\n    )\n\n    assert res.status_code == 200\n    assert len(res.json) == 1\n    uuid = res.json[0]\n\n    # Verify the configuration was applied\n    watch = live_server.app.config['DATASTORE'].data['watching'][uuid]\n    assert watch['include_filters'] == ['div.content'], \"include_filters should be set as array\"\n    assert watch['paused'] == True, \"paused should be True\"\n    assert watch['processor'] == 'text_json_diff', \"processor should be set\"\n    assert watch['title'] == 'Imported with Config', \"title should be set\"\n\n    # Test 3: Import with array field (notification_urls) - using valid Apprise format\n    params = urllib.parse.urlencode({\n        'tag': 'notification-test',\n        'notification_urls': 'mailto://test@example.com,mailto://admin@example.com'\n    })\n\n    res = client.post(\n        url_for(\"import\") + \"?\" + params,\n        data='https://website4.com',\n        headers={'x-api-key': api_key},\n        follow_redirects=True\n    )\n\n    assert res.status_code == 200\n    uuid = res.json[0]\n    watch = live_server.app.config['DATASTORE'].data['watching'][uuid]\n    assert isinstance(watch['notification_urls'], list), \"notification_urls must be stored as a list\"\n    assert len(watch['notification_urls']) == 2, \"notification_urls should have 2 entries\"\n    assert 'mailto://test@example.com' in watch['notification_urls'], \"notification_urls should contain first email\"\n    assert 'mailto://admin@example.com' in watch['notification_urls'], \"notification_urls should contain second email\"\n\n    # Test 4: Import with object field (time_between_check)\n    import json\n    time_config = json.dumps({\"hours\": 2, \"minutes\": 30})\n    params = urllib.parse.urlencode({\n        'tag': 'schedule-test',\n        'time_between_check': time_config\n    })\n\n    res = client.post(\n        url_for(\"import\") + \"?\" + params,\n        data='https://website5.com',\n        headers={'x-api-key': api_key},\n        follow_redirects=True\n    )\n\n    assert res.status_code == 200\n    uuid = res.json[0]\n    watch = live_server.app.config['DATASTORE'].data['watching'][uuid]\n    assert watch['time_between_check']['hours'] == 2, \"time_between_check hours should be 2\"\n    assert watch['time_between_check']['minutes'] == 30, \"time_between_check minutes should be 30\"\n\n    # Test 5: Import with invalid processor (should fail)\n    res = client.post(\n        url_for(\"import\") + \"?processor=invalid_processor\",\n        data='https://website6.com',\n        headers={'x-api-key': api_key},\n        follow_redirects=True\n    )\n\n    assert res.status_code == 400, \"Should reject invalid processor\"\n    assert b\"Invalid processor\" in res.data, \"Error message should mention invalid processor\"\n\n    # Test 6: Import with invalid field (should fail)\n    res = client.post(\n        url_for(\"import\") + \"?unknown_field=value\",\n        data='https://website7.com',\n        headers={'x-api-key': api_key},\n        follow_redirects=True\n    )\n\n    assert res.status_code == 400, \"Should reject unknown field\"\n    assert b\"Unknown watch configuration parameter\" in res.data, \"Error message should mention unknown parameter\"\n\n    # Test 7: Import with complex nested array (browser_steps) - array of objects\n    browser_steps = json.dumps([\n        {\"operation\": \"wait\", \"selector\": \"5\", \"optional_value\": \"\"},\n        {\"operation\": \"click\", \"selector\": \"button.submit\", \"optional_value\": \"\"}\n    ])\n    params = urllib.parse.urlencode({\n        'tag': 'browser-test',\n        'browser_steps': browser_steps\n    })\n\n    res = client.post(\n        url_for(\"import\") + \"?\" + params,\n        data='https://website8.com',\n        headers={'x-api-key': api_key},\n        follow_redirects=True\n    )\n\n    assert res.status_code == 200, \"Should accept browser_steps array\"\n    uuid = res.json[0]\n    watch = live_server.app.config['DATASTORE'].data['watching'][uuid]\n    assert len(watch['browser_steps']) == 2, \"Should have 2 browser steps\"\n    assert watch['browser_steps'][0]['operation'] == 'wait', \"First step should be wait\"\n    assert watch['browser_steps'][1]['operation'] == 'click', \"Second step should be click\"\n    assert watch['browser_steps'][1]['selector'] == 'button.submit', \"Second step selector should be button.submit\"\n\n    # Cleanup\n    delete_all_watches(client)\n\n\ndef test_api_import_small_synchronous(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"Test that small imports (< threshold) are processed synchronously\"\"\"\n    from changedetectionio.api.Import import IMPORT_SWITCH_TO_BACKGROUND_THRESHOLD\n\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n\n    # Use local test endpoint to avoid network delays\n    test_url_base = url_for('test_endpoint', _external=True)\n\n    # Create URLs: threshold - 1 to stay under limit\n    num_urls = min(5, IMPORT_SWITCH_TO_BACKGROUND_THRESHOLD - 1)  # Use small number for faster test\n    urls = '\\n'.join([f'{test_url_base}?id=small-{i}' for i in range(num_urls)])\n\n    # Import small batch\n    res = client.post(\n        url_for(\"import\") + \"?tag=small-test\",\n        data=urls,\n        headers={'x-api-key': api_key},\n        follow_redirects=True\n    )\n\n    # Should return 200 OK with UUID list (synchronous)\n    assert res.status_code == 200, f\"Should return 200 for small imports, got {res.status_code}\"\n    assert isinstance(res.json, list), \"Response should be a list of UUIDs\"\n    assert len(res.json) == num_urls, f\"Should return {num_urls} UUIDs, got {len(res.json)}\"\n\n    # Verify all watches were created immediately\n    for uuid in res.json:\n        assert uuid in live_server.app.config['DATASTORE'].data['watching'], \\\n            f\"Watch {uuid} should exist immediately after synchronous import\"\n\n    print(f\"\\n✓ Successfully created {num_urls} watches synchronously\")\n\n\ndef test_api_import_large_background(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"Test that large imports (>= threshold) are processed in background thread\"\"\"\n    from changedetectionio.api.Import import IMPORT_SWITCH_TO_BACKGROUND_THRESHOLD\n    import time\n\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n\n    # Use local test endpoint to avoid network delays\n    test_url_base = url_for('test_endpoint', _external=True)\n\n    # Create URLs: threshold + 10 to trigger background processing\n    num_urls = IMPORT_SWITCH_TO_BACKGROUND_THRESHOLD + 10\n    urls = '\\n'.join([f'{test_url_base}?id=bulk-{i}' for i in range(num_urls)])\n\n    # Import large batch\n    res = client.post(\n        url_for(\"import\") + \"?tag=bulk-test\",\n        data=urls,\n        headers={'x-api-key': api_key},\n        follow_redirects=True\n    )\n\n    # Should return 202 Accepted (background processing)\n    assert res.status_code == 202, f\"Should return 202 for large imports, got {res.status_code}\"\n    assert b\"background\" in res.data.lower(), \"Response should mention background processing\"\n\n    # Extract expected count from response\n    response_json = res.json\n    assert 'count' in response_json, \"Response should include count\"\n    assert response_json['count'] == num_urls, f\"Count should be {num_urls}, got {response_json['count']}\"\n\n    # Wait for background thread to complete (with timeout)\n    max_wait = 10  # seconds\n    wait_interval = 0.5\n    elapsed = 0\n    watches_created = 0\n\n    while elapsed < max_wait:\n        time.sleep(wait_interval)\n        elapsed += wait_interval\n\n        # Count how many watches have been created\n        watches_created = len([\n            uuid for uuid, watch in live_server.app.config['DATASTORE'].data['watching'].items()\n            if 'id=bulk-' in watch['url']\n        ])\n\n        if watches_created == num_urls:\n            break\n\n    # Verify all watches were created\n    assert watches_created == num_urls, \\\n        f\"Expected {num_urls} watches to be created, but found {watches_created} after {elapsed}s\"\n\n    # Verify watches have correct configuration\n    bulk_watches = [\n        watch for watch in live_server.app.config['DATASTORE'].data['watching'].values()\n        if 'id=bulk-' in watch['url']\n    ]\n\n    assert len(bulk_watches) == num_urls, \"All bulk watches should exist\"\n\n    # Check that they have the correct tag\n    datastore = live_server.app.config['DATASTORE']\n    # Get UUIDs of bulk watches by filtering the datastore keys\n    bulk_watch_uuids = [\n        uuid for uuid, watch in live_server.app.config['DATASTORE'].data['watching'].items()\n        if 'id=bulk-' in watch['url']\n    ]\n    for watch_uuid in bulk_watch_uuids:\n        tags = datastore.get_all_tags_for_watch(uuid=watch_uuid)\n        tag_names = [t['title'] for t in tags.values()]\n        assert 'bulk-test' in tag_names, f\"Watch {watch_uuid} should have 'bulk-test' tag\"\n\n    print(f\"\\n✓ Successfully created {num_urls} watches in background (took {elapsed}s)\")\n\n\ndef test_api_restock_processor_config(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test that processor_config_restock_diff is accepted by the API for watches using\n    restock_diff processor, that its schema is validated (enum values, types), and that\n    genuinely unknown fields are rejected with an error that originates from the\n    OpenAPI spec validation layer.\n    \"\"\"\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n    test_url = url_for('test_endpoint', _external=True)\n\n    # Create a watch in restock_diff mode WITH processor_config in the POST body (matches the API docs example)\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\n            \"url\": test_url,\n            \"processor\": \"restock_diff\",\n            \"title\": \"Restock test\",\n            \"processor_config_restock_diff\": {\n                \"in_stock_processing\": \"in_stock_only\",\n                \"follow_price_changes\": True,\n                \"price_change_min\": 8888888.0,\n            }\n        }),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n        follow_redirects=True\n    )\n    assert res.status_code == 201\n    watch_uuid = res.json.get('uuid')\n    assert is_valid_uuid(watch_uuid)\n\n    # Verify the value set on POST is reflected in the UI edit page (not just via PUT)\n    res = client.get(url_for(\"ui.ui_edit.edit_page\", uuid=watch_uuid))\n    assert res.status_code == 200\n    assert b'8888888' in res.data, \"price_change_min set via POST should appear in the UI edit form\"\n\n    # Valid processor_config_restock_diff update via PUT should also be accepted\n    res = client.put(\n        url_for(\"watch\", uuid=watch_uuid),\n        headers={'x-api-key': api_key, 'content-type': 'application/json'},\n        data=json.dumps({\n            \"processor_config_restock_diff\": {\n                \"in_stock_processing\": \"all_changes\",\n                \"follow_price_changes\": False,\n                \"price_change_min\": 8888888.0,\n                \"price_change_max\": 9999999.0,\n            }\n        }),\n    )\n    assert res.status_code == 200, f\"Valid processor_config_restock_diff should be accepted, got: {res.data}\"\n\n    # Verify the updated value is still reflected in the UI edit page\n    res = client.get(url_for(\"ui.ui_edit.edit_page\", uuid=watch_uuid))\n    assert res.status_code == 200\n    assert b'8888888' in res.data, \"price_change_min set via PUT should appear in the UI edit form\"\n\n    # An invalid enum value inside processor_config_restock_diff should be rejected by the spec\n    res = client.put(\n        url_for(\"watch\", uuid=watch_uuid),\n        headers={'x-api-key': api_key, 'content-type': 'application/json'},\n        data=json.dumps({\n            \"processor_config_restock_diff\": {\n                \"in_stock_processing\": \"not_a_valid_enum_value\"\n            }\n        }),\n    )\n    assert res.status_code == 400, \"Invalid enum value in processor config should be rejected\"\n    assert b'Validation failed' in res.data, \"Rejection should come from OpenAPI spec validation layer\"\n\n    # A completely unknown field should be rejected (either by OpenAPI spec validation or\n    # the application-level field filter — both are acceptable gatekeepers)\n    res = client.put(\n        url_for(\"watch\", uuid=watch_uuid),\n        headers={'x-api-key': api_key, 'content-type': 'application/json'},\n        data=json.dumps({\"field_that_is_not_in_the_spec_at_all\": \"some value\"}),\n    )\n    assert res.status_code == 400, \"Unknown fields should be rejected\"\n    assert (b'Validation failed' in res.data or b'Unknown field' in res.data), \\\n        \"Rejection should come from either the OpenAPI spec validation layer or application field filter\"\n\n    delete_all_watches(client)\n\n\ndef test_api_conflict_UI_password(client, live_server, measure_memory_usage, datastore_path):\n\n\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n\n    # Enable password check and diff page access bypass\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\"application-password\": \"foobar\", # password is now set! API should still work!\n              \"application-api_access_token_enabled\": \"y\",\n              \"requests-time_between_check-minutes\": 180,\n              'application-fetch_backend': \"html_requests\"},\n        follow_redirects=True\n    )\n\n    assert b\"Password protection enabled.\" in res.data\n\n    # Create a watch\n    set_original_response(datastore_path=datastore_path)\n    test_url = url_for('test_endpoint', _external=True)\n\n    # Create new\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\"url\": test_url, \"title\": \"My test URL\" }),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n        follow_redirects=True\n    )\n\n    assert res.status_code == 201\n\n\n    wait_for_all_checks(client)\n    url = url_for(\"createwatch\")\n    # Get a listing, it will be the first one\n    res = client.get(\n        url,\n        headers={'x-api-key': api_key}\n    )\n    assert res.status_code == 200\n\n    assert len(res.json)\n\n\ndef test_api_url_validation(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test URL validation for edge cases in both CREATE and UPDATE endpoints.\n    Addresses security issues where empty/null/invalid URLs could bypass validation.\n\n    This test ensures that:\n    - CREATE endpoint rejects null, empty, and invalid URLs\n    - UPDATE endpoint rejects attempts to change URL to null, empty, or invalid\n    - UPDATE endpoint allows updating other fields without touching URL\n    - URL validation properly checks protocol, format, and safety\n    \"\"\"\n\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n    set_original_response(datastore_path=datastore_path)\n    test_url = url_for('test_endpoint', _external=True)\n\n    # Test 1: CREATE with null URL should fail\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\"url\": None}),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n        follow_redirects=True\n    )\n    assert res.status_code == 400, \"Creating watch with null URL should fail\"\n\n    # Test 2: CREATE with empty string URL should fail\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\"url\": \"\"}),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n        follow_redirects=True\n    )\n    assert res.status_code == 400, \"Creating watch with empty string URL should fail\"\n    assert b'Invalid or unsupported URL' in res.data or b'required' in res.data.lower()\n\n    # Test 3: CREATE with whitespace-only URL should fail\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\"url\": \"   \"}),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n        follow_redirects=True\n    )\n    assert res.status_code == 400, \"Creating watch with whitespace-only URL should fail\"\n\n    # Test 4: CREATE with invalid protocol should fail\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\"url\": \"javascript:alert(1)\"}),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n        follow_redirects=True\n    )\n    assert res.status_code == 400, \"Creating watch with javascript: protocol should fail\"\n\n    # Test 5: CREATE with missing protocol should fail\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\"url\": \"example.com\"}),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n        follow_redirects=True\n    )\n    assert res.status_code == 400, \"Creating watch without protocol should fail\"\n\n    # Test 6: CREATE with valid URL should succeed (baseline)\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\"url\": test_url, \"title\": \"Valid URL test\"}),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n        follow_redirects=True\n    )\n    assert res.status_code == 201, \"Creating watch with valid URL should succeed\"\n    assert is_valid_uuid(res.json.get('uuid'))\n    watch_uuid = res.json.get('uuid')\n    wait_for_all_checks(client)\n\n    # Test 7: UPDATE to null URL should fail\n    res = client.put(\n        url_for(\"watch\", uuid=watch_uuid),\n        headers={'x-api-key': api_key, 'content-type': 'application/json'},\n        data=json.dumps({\"url\": None}),\n    )\n    assert res.status_code == 400, \"Updating watch URL to null should fail\"\n    # Accept either OpenAPI validation error or our custom validation error\n    assert (b'URL cannot be null' in res.data or\n            b'Validation failed' in res.data or\n            b'validation error' in res.data.lower())\n\n    # Test 8: UPDATE to empty string URL should fail\n    res = client.put(\n        url_for(\"watch\", uuid=watch_uuid),\n        headers={'x-api-key': api_key, 'content-type': 'application/json'},\n        data=json.dumps({\"url\": \"\"}),\n    )\n    assert res.status_code == 400, \"Updating watch URL to empty string should fail\"\n    # Accept either our custom validation error or OpenAPI/schema validation error\n    assert b'URL cannot be empty' in res.data or b'OpenAPI validation' in res.data or b'Invalid or unsupported URL' in res.data\n\n    # Test 9: UPDATE to whitespace-only URL should fail\n    res = client.put(\n        url_for(\"watch\", uuid=watch_uuid),\n        headers={'x-api-key': api_key, 'content-type': 'application/json'},\n        data=json.dumps({\"url\": \"   \\t\\n  \"}),\n    )\n    assert res.status_code == 400, \"Updating watch URL to whitespace should fail\"\n    # Accept either our custom validation error or generic validation error\n    assert b'URL cannot be empty' in res.data or b'Invalid or unsupported URL' in res.data or b'validation' in res.data.lower()\n\n    # Test 10: UPDATE to invalid protocol should fail (javascript:)\n    res = client.put(\n        url_for(\"watch\", uuid=watch_uuid),\n        headers={'x-api-key': api_key, 'content-type': 'application/json'},\n        data=json.dumps({\"url\": \"javascript:alert(document.domain)\"}),\n    )\n    assert res.status_code == 400, \"Updating watch URL to XSS attempt should fail\"\n    assert b'Invalid or unsupported URL' in res.data or b'protocol' in res.data.lower()\n\n    # Test 11: UPDATE to file:// protocol should fail (unless ALLOW_FILE_URI is set)\n    res = client.put(\n        url_for(\"watch\", uuid=watch_uuid),\n        headers={'x-api-key': api_key, 'content-type': 'application/json'},\n        data=json.dumps({\"url\": \"file:///etc/passwd\"}),\n    )\n    assert res.status_code == 400, \"Updating watch URL to file:// should fail by default\"\n\n    # Test 12: UPDATE other fields without URL should succeed\n    res = client.put(\n        url_for(\"watch\", uuid=watch_uuid),\n        headers={'x-api-key': api_key, 'content-type': 'application/json'},\n        data=json.dumps({\"title\": \"Updated title without URL change\"}),\n    )\n    assert res.status_code == 200, \"Updating other fields without URL should succeed\"\n\n    # Test 13: Verify URL is still valid after non-URL update\n    res = client.get(\n        url_for(\"watch\", uuid=watch_uuid),\n        headers={'x-api-key': api_key}\n    )\n    assert res.json.get('url') == test_url, \"URL should remain unchanged\"\n    assert res.json.get('title') == \"Updated title without URL change\"\n\n    # Test 14: UPDATE to valid different URL should succeed\n    new_valid_url = test_url + \"?new=param\"\n    res = client.put(\n        url_for(\"watch\", uuid=watch_uuid),\n        headers={'x-api-key': api_key, 'content-type': 'application/json'},\n        data=json.dumps({\"url\": new_valid_url}),\n    )\n    assert res.status_code == 200, \"Updating to valid different URL should succeed\"\n\n    # Test 15: Verify URL was actually updated\n    res = client.get(\n        url_for(\"watch\", uuid=watch_uuid),\n        headers={'x-api-key': api_key}\n    )\n    assert res.json.get('url') == new_valid_url, \"URL should be updated to new valid URL\"\n\n    # Test 16: CREATE with XSS in URL parameters should fail\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\"url\": \"http://example.com?xss=<script>alert(1)</script>\"}),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n        follow_redirects=True\n    )\n    # This should fail because of suspicious characters check\n    assert res.status_code == 400, \"Creating watch with XSS in URL params should fail\"\n\n    # Cleanup\n    client.delete(\n        url_for(\"watch\", uuid=watch_uuid),\n        headers={'x-api-key': api_key},\n    )\n    delete_all_watches(client)\n\n\ndef test_api_time_between_check_validation(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test that time_between_check validation works correctly:\n    - When time_between_check_use_default is false, at least one time value must be > 0\n    - Values must be valid integers\n    \"\"\"\n    import json\n    from flask import url_for\n    \n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n    \n    # Test 1: time_between_check_use_default=false with NO time_between_check should fail\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\n            \"url\": \"https://example.com\",\n            \"time_between_check_use_default\": False\n        }),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    assert res.status_code == 400, \"Should fail when time_between_check_use_default=false with no time_between_check\"\n    assert b\"At least one time interval\" in res.data, \"Error message should mention time interval requirement\"\n    \n    # Test 2: time_between_check_use_default=false with ALL zeros should fail\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\n            \"url\": \"https://example.com\",\n            \"time_between_check_use_default\": False,\n            \"time_between_check\": {\n                \"weeks\": 0,\n                \"days\": 0,\n                \"hours\": 0,\n                \"minutes\": 0,\n                \"seconds\": 0\n            }\n        }),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    assert res.status_code == 400, \"Should fail when all time values are 0\"\n    assert b\"At least one time interval\" in res.data, \"Error message should mention time interval requirement\"\n    \n    # Test 3: time_between_check_use_default=false with NULL values should fail\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\n            \"url\": \"https://example.com\",\n            \"time_between_check_use_default\": False,\n            \"time_between_check\": {\n                \"weeks\": None,\n                \"days\": None,\n                \"hours\": None,\n                \"minutes\": None,\n                \"seconds\": None\n            }\n        }),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    assert res.status_code == 400, \"Should fail when all time values are null\"\n    assert b\"At least one time interval\" in res.data, \"Error message should mention time interval requirement\"\n    \n    # Test 4: time_between_check_use_default=false with valid hours should succeed\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\n            \"url\": \"https://example.com\",\n            \"time_between_check_use_default\": False,\n            \"time_between_check\": {\n                \"hours\": 2\n            }\n        }),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    assert res.status_code == 201, \"Should succeed with valid hours value\"\n    uuid1 = res.json.get('uuid')\n    \n    # Test 5: time_between_check_use_default=false with valid minutes should succeed\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\n            \"url\": \"https://example2.com\",\n            \"time_between_check_use_default\": False,\n            \"time_between_check\": {\n                \"minutes\": 30\n            }\n        }),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    assert res.status_code == 201, \"Should succeed with valid minutes value\"\n    uuid2 = res.json.get('uuid')\n    \n    # Test 6: time_between_check_use_default=true (or missing) with no time_between_check should succeed (uses defaults)\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\n            \"url\": \"https://example3.com\",\n            \"time_between_check_use_default\": True\n        }),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    assert res.status_code == 201, \"Should succeed when using default settings\"\n    uuid3 = res.json.get('uuid')\n    \n    # Test 7: Default behavior (no time_between_check_use_default field) should use defaults and succeed\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\n            \"url\": \"https://example4.com\"\n        }),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    assert res.status_code == 201, \"Should succeed with default behavior (using global settings)\"\n    uuid4 = res.json.get('uuid')\n    \n    # Test 8: Verify integer type validation - string should fail (OpenAPI validation)\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\n            \"url\": \"https://example5.com\",\n            \"time_between_check_use_default\": False,\n            \"time_between_check\": {\n                \"hours\": \"not_a_number\"\n            }\n        }),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    assert res.status_code == 400, \"Should fail when time value is not an integer\"\n    assert b\"Validation failed\" in res.data or b\"not of type\" in res.data, \"Should mention validation/type error\"\n    \n    # Cleanup\n    for uuid in [uuid1, uuid2, uuid3, uuid4]:\n        client.delete(\n            url_for(\"watch\", uuid=uuid),\n            headers={'x-api-key': api_key},\n        )\n"
  },
  {
    "path": "changedetectionio/tests/test_api_notification_urls_validation.py",
    "content": "#!/usr/bin/env python3\n\n\"\"\"\nTest notification_urls validation in Watch and Tag API endpoints.\nEnsures that invalid AppRise URLs are rejected when setting notification_urls.\n\nValid AppRise notification URLs use specific protocols like:\n- posts://example.com - POST to HTTP endpoint\n- gets://example.com - GET to HTTP endpoint\n- mailto://user@example.com - Email\n- slack://token/channel - Slack\n- discord://webhook_id/webhook_token - Discord\n- etc.\n\nInvalid notification URLs:\n- https://example.com - Plain HTTPS is NOT a valid AppRise notification protocol\n- ftp://example.com - FTP is NOT a valid AppRise notification protocol\n- Plain URLs without proper AppRise protocol prefix\n\"\"\"\n\nfrom flask import url_for\nimport json\n\n\ndef test_watch_notification_urls_validation(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"Test that Watch PUT/POST endpoints validate notification_urls.\"\"\"\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n\n    # Test 1: Create a watch with valid notification URLs\n    valid_urls = [\"posts://example.com/notify1\", \"posts://example.com/notify2\"]\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\n            \"url\": \"https://example.com\",\n            \"notification_urls\": valid_urls\n        }),\n        headers={'content-type': 'application/json', 'x-api-key': api_key}\n    )\n    assert res.status_code == 201, \"Should accept valid notification URLs on watch creation\"\n    watch_uuid = res.json['uuid']\n\n    # Verify the notification URLs were saved\n    res = client.get(\n        url_for(\"watch\", uuid=watch_uuid),\n        headers={'x-api-key': api_key}\n    )\n    assert res.status_code == 200\n    assert set(res.json['notification_urls']) == set(valid_urls), \"Valid notification URLs should be saved\"\n\n    # Test 2: Try to create a watch with invalid notification URLs (https:// is not valid)\n    invalid_urls = [\"https://example.com/webhook\"]\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\n            \"url\": \"https://example.com\",\n            \"notification_urls\": invalid_urls\n        }),\n        headers={'content-type': 'application/json', 'x-api-key': api_key}\n    )\n    assert res.status_code == 400, \"Should reject https:// notification URLs (not a valid AppRise protocol)\"\n    assert b\"is not a valid AppRise URL\" in res.data, \"Should provide AppRise validation error message\"\n\n    # Test 2b: Also test other invalid protocols\n    invalid_urls_ftp = [\"ftp://not-apprise-url\"]\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\n            \"url\": \"https://example.com\",\n            \"notification_urls\": invalid_urls_ftp\n        }),\n        headers={'content-type': 'application/json', 'x-api-key': api_key}\n    )\n    assert res.status_code == 400, \"Should reject ftp:// notification URLs\"\n    assert b\"is not a valid AppRise URL\" in res.data, \"Should provide AppRise validation error message\"\n\n    # Test 3: Update watch with valid notification URLs\n    new_valid_urls = [\"posts://newserver.com\"]\n    res = client.put(\n        url_for(\"watch\", uuid=watch_uuid),\n        data=json.dumps({\"notification_urls\": new_valid_urls}),\n        headers={'content-type': 'application/json', 'x-api-key': api_key}\n    )\n    assert res.status_code == 200, \"Should accept valid notification URLs on watch update\"\n\n    # Verify the notification URLs were updated\n    res = client.get(\n        url_for(\"watch\", uuid=watch_uuid),\n        headers={'x-api-key': api_key}\n    )\n    assert res.status_code == 200\n    assert res.json['notification_urls'] == new_valid_urls, \"Valid notification URLs should be updated\"\n\n    # Test 4: Try to update watch with invalid notification URLs (plain https:// not valid)\n    invalid_https_url = [\"https://example.com/webhook\"]\n    res = client.put(\n        url_for(\"watch\", uuid=watch_uuid),\n        data=json.dumps({\"notification_urls\": invalid_https_url}),\n        headers={'content-type': 'application/json', 'x-api-key': api_key}\n    )\n    assert res.status_code == 400, \"Should reject https:// notification URLs on watch update\"\n    assert b\"is not a valid AppRise URL\" in res.data, \"Should provide AppRise validation error message\"\n\n    # Test 5: Update watch with non-list notification_urls (caught by OpenAPI schema validation)\n    res = client.put(\n        url_for(\"watch\", uuid=watch_uuid),\n        data=json.dumps({\"notification_urls\": \"not-a-list\"}),\n        headers={'content-type': 'application/json', 'x-api-key': api_key}\n    )\n    assert res.status_code == 400, \"Should reject non-list notification_urls\"\n    assert b\"Validation failed\" in res.data or b\"is not of type\" in res.data\n\n    # Test 6: Verify original URLs are preserved after failed update\n    res = client.get(\n        url_for(\"watch\", uuid=watch_uuid),\n        headers={'x-api-key': api_key}\n    )\n    assert res.status_code == 200\n    assert res.json['notification_urls'] == new_valid_urls, \"URLs should remain unchanged after validation failure\"\n\n\ndef test_tag_notification_urls_validation(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"Test that Tag PUT endpoint validates notification_urls.\"\"\"\n    from changedetectionio.model import Tag\n\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n    datastore = live_server.app.config['DATASTORE']\n\n    # Create a tag\n    tag_uuid = datastore.add_tag(title=\"Test Tag\")\n    assert tag_uuid is not None\n\n    # Test 1: Update tag with valid notification URLs\n    valid_urls = [\"posts://example.com/tag-notify\"]\n    res = client.put(\n        url_for(\"tag\", uuid=tag_uuid),\n        data=json.dumps({\"notification_urls\": valid_urls}),\n        headers={'content-type': 'application/json', 'x-api-key': api_key}\n    )\n    assert res.status_code == 200, \"Should accept valid notification URLs on tag update\"\n\n    # Verify the notification URLs were saved\n    tag = datastore.data['settings']['application']['tags'][tag_uuid]\n    assert tag['notification_urls'] == valid_urls, \"Valid notification URLs should be saved to tag\"\n\n    # Test 2: Try to update tag with invalid notification URLs (https:// not valid)\n    invalid_urls = [\"https://example.com/webhook\"]\n    res = client.put(\n        url_for(\"tag\", uuid=tag_uuid),\n        data=json.dumps({\"notification_urls\": invalid_urls}),\n        headers={'content-type': 'application/json', 'x-api-key': api_key}\n    )\n    assert res.status_code == 400, \"Should reject https:// notification URLs on tag update\"\n    assert b\"is not a valid AppRise URL\" in res.data, \"Should provide AppRise validation error message\"\n\n    # Test 3: Update tag with non-list notification_urls (caught by OpenAPI schema validation)\n    res = client.put(\n        url_for(\"tag\", uuid=tag_uuid),\n        data=json.dumps({\"notification_urls\": \"not-a-list\"}),\n        headers={'content-type': 'application/json', 'x-api-key': api_key}\n    )\n    assert res.status_code == 400, \"Should reject non-list notification_urls\"\n    assert b\"Validation failed\" in res.data or b\"is not of type\" in res.data\n\n    # Test 4: Verify original URLs are preserved after failed update\n    tag = datastore.data['settings']['application']['tags'][tag_uuid]\n    assert tag['notification_urls'] == valid_urls, \"URLs should remain unchanged after validation failure\"\n"
  },
  {
    "path": "changedetectionio/tests/test_api_notifications.py",
    "content": "#!/usr/bin/env python3\n\nfrom flask import url_for\nfrom .util import live_server_setup\nimport json\n\ndef test_api_notifications_crud(client, live_server, measure_memory_usage, datastore_path):\n   #  live_server_setup(live_server) # Setup on conftest per function\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n\n    # Confirm notifications are initially empty\n    res = client.get(\n        url_for(\"notifications\"),\n        headers={'x-api-key': api_key}\n    )\n    assert res.status_code == 200\n    assert res.json == {\"notification_urls\": []}\n\n    # Add notification URLs\n    test_urls = [\"posts://example.com/notify1\", \"posts://example.com/notify2\"]\n    res = client.post(\n        url_for(\"notifications\"),\n        data=json.dumps({\"notification_urls\": test_urls}),\n        headers={'content-type': 'application/json', 'x-api-key': api_key}\n    )\n    assert res.status_code == 201\n    for url in test_urls:\n        assert url in res.json[\"notification_urls\"]\n\n    # Confirm the notification URLs were added\n    res = client.get(\n        url_for(\"notifications\"),\n        headers={'x-api-key': api_key}\n    )\n    assert res.status_code == 200\n    for url in test_urls:\n        assert url in res.json[\"notification_urls\"]\n\n    # Delete one notification URL\n    res = client.delete(\n        url_for(\"notifications\"),\n        data=json.dumps({\"notification_urls\": [test_urls[0]]}),\n        headers={'content-type': 'application/json', 'x-api-key': api_key}\n    )\n    assert res.status_code == 204\n\n    # Confirm it was removed and the other remains\n    res = client.get(\n        url_for(\"notifications\"),\n        headers={'x-api-key': api_key}\n    )\n    assert res.status_code == 200\n    assert test_urls[0] not in res.json[\"notification_urls\"]\n    assert test_urls[1] in res.json[\"notification_urls\"]\n\n    # Try deleting a non-existent URL\n    res = client.delete(\n        url_for(\"notifications\"),\n        data=json.dumps({\"notification_urls\": [\"posts://nonexistent.com\"]}),\n        headers={'content-type': 'application/json', 'x-api-key': api_key}\n    )\n    assert res.status_code == 400\n\n    res = client.post(\n        url_for(\"notifications\"),\n        data=json.dumps({\"notification_urls\": test_urls}),\n        headers={'content-type': 'application/json', 'x-api-key': api_key}\n    )\n    assert res.status_code == 201\n\n    # Replace with a new list\n    replacement_urls = [\"posts://new.example.com\"]\n    res = client.put(\n        url_for(\"notifications\"),\n        data=json.dumps({\"notification_urls\": replacement_urls}),\n        headers={'content-type': 'application/json', 'x-api-key': api_key}\n    )\n    assert res.status_code == 200\n    assert res.json[\"notification_urls\"] == replacement_urls\n\n    # Replace with an empty list\n    res = client.put(\n        url_for(\"notifications\"),\n        data=json.dumps({\"notification_urls\": []}),\n        headers={'content-type': 'application/json', 'x-api-key': api_key}\n    )\n    assert res.status_code == 200\n    assert res.json[\"notification_urls\"] == []\n\n    # Provide an invalid AppRise URL to trigger validation error\n    invalid_urls = [\"ftp://not-app-rise\"]\n    res = client.post(\n        url_for(\"notifications\"),\n        data=json.dumps({\"notification_urls\": invalid_urls}),\n        headers={'content-type': 'application/json', 'x-api-key': api_key}\n    )\n    assert res.status_code == 400\n    assert \"is not a valid AppRise URL.\" in res.data.decode()\n\n    res = client.put(\n        url_for(\"notifications\"),\n        data=json.dumps({\"notification_urls\": invalid_urls}),\n        headers={'content-type': 'application/json', 'x-api-key': api_key}\n    )\n    assert res.status_code == 400\n    assert \"is not a valid AppRise URL.\" in res.data.decode()\n\n    "
  },
  {
    "path": "changedetectionio/tests/test_api_openapi.py",
    "content": "#!/usr/bin/env python3\n\"\"\"\nOpenAPI validation tests for ChangeDetection.io API\n\nThis test file specifically verifies that OpenAPI validation is working correctly\nby testing various scenarios that should trigger validation errors.\n\"\"\"\n\nimport time\nimport json\nfrom flask import url_for\nfrom .util import live_server_setup, wait_for_all_checks, delete_all_watches\n\n\ndef test_openapi_merged_spec_contains_restock_fields():\n    \"\"\"\n    Unit test: verify that build_merged_spec_dict() correctly merges the\n    restock_diff processor api.yaml into the base spec so that\n    WatchBase.properties includes processor_config_restock_diff with all\n    expected sub-fields.  No live server required.\n    \"\"\"\n    from changedetectionio.api import build_merged_spec_dict\n\n    spec = build_merged_spec_dict()\n    schemas = spec['components']['schemas']\n\n    # The merged schema for processor_config_restock_diff should exist\n    assert 'processor_config_restock_diff' in schemas, \\\n        \"processor_config_restock_diff schema missing from merged spec\"\n\n    restock_schema = schemas['processor_config_restock_diff']\n    props = restock_schema.get('properties', {})\n\n    expected_fields = {\n        'in_stock_processing',\n        'follow_price_changes',\n        'price_change_min',\n        'price_change_max',\n        'price_change_threshold_percent',\n    }\n    missing = expected_fields - set(props.keys())\n    assert not missing, f\"Missing fields in processor_config_restock_diff schema: {missing}\"\n\n    # in_stock_processing must be an enum with the three valid values\n    enum_values = set(props['in_stock_processing'].get('enum', []))\n    assert enum_values == {'in_stock_only', 'all_changes', 'off'}, \\\n        f\"Unexpected enum values for in_stock_processing: {enum_values}\"\n\n    # WatchBase.properties must carry a $ref to the restock schema so the\n    # validation middleware can enforce it on every POST/PUT to /watch\n    watchbase_props = schemas['WatchBase']['properties']\n    assert 'processor_config_restock_diff' in watchbase_props, \\\n        \"processor_config_restock_diff not wired into WatchBase.properties\"\n    ref = watchbase_props['processor_config_restock_diff'].get('$ref', '')\n    assert 'processor_config_restock_diff' in ref, \\\n        f\"WatchBase.processor_config_restock_diff should $ref the schema, got: {ref}\"\n\n\ndef test_openapi_validation_invalid_content_type_on_create_watch(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"Test that creating a watch with invalid content-type triggers OpenAPI validation error.\"\"\"\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n\n    # Try to create a watch with JSON data but without proper content-type header\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\"url\": \"https://example.com\", \"title\": \"Test Watch\"}),\n        headers={'x-api-key': api_key},  # Missing 'content-type': 'application/json'\n        follow_redirects=True\n    )\n\n    # Should get 400 error due to OpenAPI validation failure\n    assert res.status_code == 400, f\"Expected 400 but got {res.status_code}\"\n    assert b\"Validation failed\" in res.data, \"Should contain validation error message\"\n    delete_all_watches(client)\n\n\ndef test_openapi_validation_missing_required_field_create_watch(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"Test that creating a watch without required URL field triggers OpenAPI validation error.\"\"\"\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n\n    # Try to create a watch without the required 'url' field\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\"title\": \"Test Watch Without URL\"}),  # Missing required 'url' field\n        headers={'x-api-key': api_key, 'content-type': 'application/json'},\n        follow_redirects=True\n    )\n\n    # Should get 400 error due to missing required field\n    assert res.status_code == 400, f\"Expected 400 but got {res.status_code}\"\n    assert b\"Validation failed\" in res.data, \"Should contain validation error message\"\n    delete_all_watches(client)\n\n\ndef test_openapi_validation_invalid_field_in_request_body(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"Test that including invalid fields triggers OpenAPI validation error.\"\"\"\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n\n    # First create a valid watch\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\"url\": \"https://example.com\", \"title\": \"Test Watch\"}),\n        headers={'x-api-key': api_key, 'content-type': 'application/json'},\n        follow_redirects=True\n    )\n    assert res.status_code == 201, \"Watch creation should succeed\"\n\n    # Get the watch list to find the UUID\n    res = client.get(\n        url_for(\"createwatch\"),\n        headers={'x-api-key': api_key}\n    )\n    assert res.status_code == 200\n    watch_uuid = list(res.json.keys())[0]\n\n    # Now try to update the watch with an invalid field\n    res = client.put(\n        url_for(\"watch\", uuid=watch_uuid),\n        headers={'x-api-key': api_key, 'content-type': 'application/json'},\n        data=json.dumps({\n            \"title\": \"Updated title\",\n            \"invalid_field_that_doesnt_exist\": \"this should cause validation error\"\n        }),\n    )\n\n    # Should get 400 error due to invalid field (this will be caught by internal validation)\n    # Note: This tests the flow where OpenAPI validation passes but internal validation catches it\n    assert res.status_code == 400, f\"Expected 400 but got {res.status_code}\"\n    # Backend validation now returns \"Unknown field(s):\" message\n    assert b\"Unknown field\" in res.data, \\\n            \"Should contain validation error about unknown fields\"\n    delete_all_watches(client)\n\n\ndef test_openapi_validation_import_wrong_content_type(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"Test that import endpoint with wrong content-type triggers OpenAPI validation error.\"\"\"\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n\n    # Try to import URLs with JSON content-type instead of text/plain\n    res = client.post(\n        url_for(\"import\") + \"?tag=test-import\",\n        data='https://website1.com\\nhttps://website2.com',\n        headers={'x-api-key': api_key, 'content-type': 'application/json'},  # Wrong content-type\n        follow_redirects=True\n    )\n\n    # Should get 400 error due to content-type mismatch\n    assert res.status_code == 400, f\"Expected 400 but got {res.status_code}\"\n    assert b\"Validation failed\" in res.data, \"Should contain validation error message\"\n    delete_all_watches(client)\n\n\ndef test_openapi_validation_import_correct_content_type_succeeds(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"Test that import endpoint with correct content-type succeeds (positive test).\"\"\"\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n\n    # Import URLs with correct text/plain content-type\n    res = client.post(\n        url_for(\"import\") + \"?tag=test-import\",\n        data='https://website1.com\\nhttps://website2.com',\n        headers={'x-api-key': api_key, 'content-type': 'text/plain'},  # Correct content-type\n        follow_redirects=True\n    )\n\n    # Should succeed\n    assert res.status_code == 200, f\"Expected 200 but got {res.status_code}\"\n    assert len(res.json) == 2, \"Should import 2 URLs\"\n    delete_all_watches(client)\n\n\ndef test_openapi_validation_get_requests_bypass_validation(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"Test that GET requests bypass OpenAPI validation entirely.\"\"\"\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n\n    # Disable API token requirement first\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\n            \"requests-time_between_check-minutes\": 180,\n            \"application-fetch_backend\": \"html_requests\",\n            \"application-api_access_token_enabled\": \"\"\n        },\n        follow_redirects=True\n    )\n    assert b\"Settings updated.\" in res.data\n\n    # Make GET request to list watches - should succeed even without API key or content-type\n    res = client.get(url_for(\"createwatch\"))  # No headers needed for GET\n    assert res.status_code == 200, f\"GET requests should succeed without OpenAPI validation, got {res.status_code}\"\n\n    # Should return JSON with watch list (empty in this case)\n    assert isinstance(res.json, dict), \"Should return JSON dictionary for watch list\"\n    delete_all_watches(client)\n\n\ndef test_openapi_validation_create_tag_missing_required_title(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"Test that creating a tag without required title triggers OpenAPI validation error.\"\"\"\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n\n    # Try to create a tag without the required 'title' field\n    res = client.post(\n        url_for(\"tag\"),\n        data=json.dumps({\"notification_urls\": [\"mailto:test@example.com\"]}),  # Missing required 'title' field\n        headers={'x-api-key': api_key, 'content-type': 'application/json'},\n        follow_redirects=True\n    )\n\n    # Should get 400 error due to missing required field\n    assert res.status_code == 400, f\"Expected 400 but got {res.status_code}\"\n    assert b\"Validation failed\" in res.data, \"Should contain validation error message\"\n    delete_all_watches(client)\n\n\ndef test_openapi_validation_watch_update_allows_partial_updates(client, live_server, measure_memory_usage, datastore_path):\n\n    \"\"\"Test that watch updates allow partial updates without requiring all fields (positive test).\"\"\"\n#xxx\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n\n    # First create a valid watch\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\"url\": \"https://example.com\", \"title\": \"Test Watch\"}),\n        headers={'x-api-key': api_key, 'content-type': 'application/json'},\n        follow_redirects=True\n    )\n    assert res.status_code == 201, \"Watch creation should succeed\"\n\n    # Get the watch list to find the UUID\n    res = client.get(\n        url_for(\"createwatch\"),\n        headers={'x-api-key': api_key}\n    )\n    assert res.status_code == 200\n    watch_uuid = list(res.json.keys())[0]\n\n    # Update only the title (partial update) - should succeed\n    res = client.put(\n        url_for(\"watch\", uuid=watch_uuid),\n        headers={'x-api-key': api_key, 'content-type': 'application/json'},\n        data=json.dumps({\"title\": \"Updated Title Only\"}),  # Only updating title, not URL\n    )\n\n    # Should succeed because UpdateWatch schema allows partial updates\n    assert res.status_code == 200, f\"Partial updates should succeed, got {res.status_code}\"\n\n    # Verify the update worked\n    res = client.get(\n        url_for(\"watch\", uuid=watch_uuid),\n        headers={'x-api-key': api_key}\n    )\n    assert res.status_code == 200\n    assert res.json.get('title') == 'Updated Title Only', \"Title should be updated\"\n    assert res.json.get('url') == 'https://example.com', \"URL should remain unchanged\"\n    delete_all_watches(client)\n"
  },
  {
    "path": "changedetectionio/tests/test_api_search.py",
    "content": "from copy import copy\n\nfrom flask import url_for\nimport json\nimport time\nfrom .util import live_server_setup, wait_for_all_checks\n\n\ndef test_api_search(client, live_server, measure_memory_usage, datastore_path):\n   #  live_server_setup(live_server) # Setup on conftest per function\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n\n    watch_data = {}\n    # Add some test watches\n    urls = [\n        'https://example.com/page1',\n        'https://example.org/testing',\n        'https://test-site.com/example'\n    ]\n\n    # Import the test URLs\n    res = client.post(\n        url_for(\"imports.import_page\"),\n        data={\"urls\": \"\\r\\n\".join(urls)},\n        follow_redirects=True\n    )\n    assert b\"3 Imported\" in res.data\n    wait_for_all_checks(client)\n\n    # Get a listing, it will be the first one\n    watches_response = client.get(\n        url_for(\"createwatch\"),\n        headers={'x-api-key': api_key}\n    )\n\n\n    # Add a title to one watch for title search testing\n    for uuid, watch in watches_response.json.items():\n\n        watch_data = client.get(url_for(\"watch\", uuid=uuid),\n                                follow_redirects=True,\n                                headers={'x-api-key': api_key}\n                                )\n\n        if urls[0] == watch_data.json['url']:\n            # HTTP PUT ( UPDATE an existing watch )\n            client.put(\n                url_for(\"watch\", uuid=uuid),\n                headers={'x-api-key': api_key, 'content-type': 'application/json'},\n                data=json.dumps({'title': 'Example Title Test'}),\n            )\n\n    # Test search by URL\n    res = client.get(url_for(\"search\")+\"?q=https://example.com/page1\", headers={'x-api-key': api_key, 'content-type': 'application/json'})\n    assert len(res.json) == 1\n    assert list(res.json.values())[0]['url'] == urls[0]\n\n    # Test search by URL - partial should NOT match without ?partial=true flag\n    res = client.get(url_for(\"search\")+\"?q=https://example\", headers={'x-api-key': api_key, 'content-type': 'application/json'})\n    assert len(res.json) == 0\n\n\n    # Test search by title\n    res = client.get(url_for(\"search\")+\"?q=Example Title Test\", headers={'x-api-key': api_key, 'content-type': 'application/json'})\n    assert len(res.json) == 1\n    assert list(res.json.values())[0]['url'] == urls[0]\n    assert list(res.json.values())[0]['title'] == 'Example Title Test'\n\n    # Test search that should return multiple results (partial = true)\n    res = client.get(url_for(\"search\")+\"?q=https://example&partial=true\", headers={'x-api-key': api_key, 'content-type': 'application/json'})\n    assert len(res.json) == 2\n\n    # Test empty search\n    res = client.get(url_for(\"search\")+\"?q=\", headers={'x-api-key': api_key, 'content-type': 'application/json'})\n    assert res.status_code == 400\n\n    # Add a tag to test search with tag filter\n    tag_name = 'test-tag'\n    res = client.post(\n        url_for(\"tag\"),\n        data=json.dumps({\"title\": tag_name}),\n        headers={'content-type': 'application/json', 'x-api-key': api_key}\n    )\n    assert res.status_code == 201\n    tag_uuid = res.json['uuid']\n\n    # Add the tag to one watch\n    for uuid, watch in watches_response.json.items():\n        if urls[2] == watch['url']:\n            client.put(\n                url_for(\"watch\", uuid=uuid),\n                headers={'x-api-key': api_key, 'content-type': 'application/json'},\n                data=json.dumps({'tags': [tag_uuid]}),\n            )\n\n\n    # Test search with tag filter and q\n    res = client.get(url_for(\"search\") + f\"?q={urls[2]}&tag={tag_name}\", headers={'x-api-key': api_key, 'content-type': 'application/json'})\n    assert len(res.json) == 1\n    assert list(res.json.values())[0]['url'] == urls[2]\n\n"
  },
  {
    "path": "changedetectionio/tests/test_api_security.py",
    "content": "#!/usr/bin/env python3\n\"\"\"\nComprehensive security and edge case tests for the API.\nTests critical areas that were identified as gaps in the existing test suite.\n\"\"\"\n\nimport time\nimport json\nimport threading\nimport uuid as uuid_module\nfrom flask import url_for\nfrom .util import live_server_setup, wait_for_all_checks, delete_all_watches\nimport os\n\n\ndef set_original_response(datastore_path):\n    test_return_data = \"\"\"<html>\n       <body>\n     Some initial text<br>\n     <p>Which is across multiple lines</p>\n     </body>\n     </html>\n    \"\"\"\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n    return None\n\n\ndef is_valid_uuid(val):\n    try:\n        uuid_module.UUID(str(val))\n        return True\n    except ValueError:\n        return False\n\n\n# ============================================================================\n# TIER 1: CRITICAL SECURITY TESTS\n# ============================================================================\n\ndef test_api_path_traversal_in_uuids(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test that path traversal attacks via UUID parameter are blocked.\n    Addresses CVE-like vulnerabilities where ../../../ in UUID could access arbitrary files.\n    \"\"\"\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n    set_original_response(datastore_path=datastore_path)\n    test_url = url_for('test_endpoint', _external=True)\n\n    # Create a valid watch first\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\"url\": test_url, \"title\": \"Valid watch\"}),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    assert res.status_code == 201\n    valid_uuid = res.json.get('uuid')\n\n    # Test 1: Path traversal with ../../../\n    res = client.get(\n        f\"/api/v1/watch/../../etc/passwd\",\n        headers={'x-api-key': api_key}\n    )\n    assert res.status_code in [400, 404], \"Path traversal should be rejected\"\n\n    # Test 2: Encoded path traversal\n    res = client.get(\n        \"/api/v1/watch/..%2F..%2F..%2Fetc%2Fpasswd\",\n        headers={'x-api-key': api_key}\n    )\n    assert res.status_code in [400, 404], \"Encoded path traversal should be rejected\"\n\n    # Test 3: Double-encoded path traversal\n    res = client.get(\n        \"/api/v1/watch/%2e%2e%2f%2e%2e%2f%2e%2e%2f\",\n        headers={'x-api-key': api_key}\n    )\n    assert res.status_code in [400, 404], \"Double-encoded traversal should be rejected\"\n\n    # Test 4: Try to access datastore file\n    res = client.get(\n        \"/api/v1/watch/../url-watches.json\",\n        headers={'x-api-key': api_key}\n    )\n    assert res.status_code in [400, 404], \"Access to datastore should be blocked\"\n\n    # Test 5: Null byte injection\n    res = client.get(\n        f\"/api/v1/watch/{valid_uuid}%00.json\",\n        headers={'x-api-key': api_key}\n    )\n    # Should either work (ignoring null byte) or reject - but not crash\n    assert res.status_code in [200, 400, 404]\n\n    # Test 6: DELETE with path traversal\n    res = client.delete(\n        \"/api/v1/watch/../../datastore/url-watches.json\",\n        headers={'x-api-key': api_key}\n    )\n    assert res.status_code in [400, 404, 405], \"DELETE with traversal should be blocked (405=method not allowed is also acceptable)\"\n\n    # Cleanup\n    client.delete(url_for(\"watch\", uuid=valid_uuid), headers={'x-api-key': api_key})\n    delete_all_watches(client)\n\n\ndef test_api_injection_via_headers_and_proxy(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test that injection attacks via headers and proxy fields are properly sanitized.\n    Addresses XSS and injection vulnerabilities.\n    \"\"\"\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n    set_original_response(datastore_path=datastore_path)\n    test_url = url_for('test_endpoint', _external=True)\n\n    # Test 1: XSS in headers\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\n            \"url\": test_url,\n            \"headers\": {\n                \"User-Agent\": \"<script>alert(1)</script>\",\n                \"X-Custom\": \"'; DROP TABLE watches; --\"\n            }\n        }),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    # Headers are metadata used for HTTP requests, not HTML rendering\n    # Storing them as-is is expected behavior\n    assert res.status_code in [201, 400]\n    if res.status_code == 201:\n        watch_uuid = res.json.get('uuid')\n        # Verify headers are stored (API returns JSON, not HTML, so no XSS risk)\n        res = client.get(url_for(\"watch\", uuid=watch_uuid), headers={'x-api-key': api_key})\n        assert res.status_code == 200\n        client.delete(url_for(\"watch\", uuid=watch_uuid), headers={'x-api-key': api_key})\n\n    # Test 2: Null bytes in headers\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\n            \"url\": test_url,\n            \"headers\": {\"X-Test\": \"value\\x00null\"}\n        }),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    # Should handle null bytes gracefully (reject or sanitize)\n    assert res.status_code in [201, 400]\n\n    # Test 3: Malformed proxy string\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\n            \"url\": test_url,\n            \"proxy\": \"http://evil.com:8080@victim.com\"\n        }),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    # Should reject invalid proxy format\n    assert res.status_code == 400\n\n    # Test 4: Control characters in notification title\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\n            \"url\": test_url,\n            \"notification_title\": \"Test\\r\\nInjected-Header: value\"\n        }),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    # Should accept but sanitize control characters\n    if res.status_code == 201:\n        watch_uuid = res.json.get('uuid')\n        client.delete(url_for(\"watch\", uuid=watch_uuid), headers={'x-api-key': api_key})\n\n    delete_all_watches(client)\n\n\ndef test_api_large_payload_dos(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test that excessively large payloads are rejected to prevent DoS.\n    Addresses memory leak issues found in changelog.\n    \"\"\"\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n    set_original_response(datastore_path=datastore_path)\n    test_url = url_for('test_endpoint', _external=True)\n\n    # Test 1: Huge ignore_text array\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\n            \"url\": test_url,\n            \"ignore_text\": [\"a\" * 10000] * 100  # 1MB of data\n        }),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    # Should either accept (with limits) or reject\n    if res.status_code == 201:\n        watch_uuid = res.json.get('uuid')\n        client.delete(url_for(\"watch\", uuid=watch_uuid), headers={'x-api-key': api_key})\n\n    # Test 2: Massive headers object\n    huge_headers = {f\"X-Header-{i}\": \"x\" * 1000 for i in range(100)}\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\n            \"url\": test_url,\n            \"headers\": huge_headers\n        }),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    # Should reject or truncate\n    assert res.status_code in [201, 400, 413]\n    if res.status_code == 201:\n        watch_uuid = res.json.get('uuid')\n        client.delete(url_for(\"watch\", uuid=watch_uuid), headers={'x-api-key': api_key})\n\n    # Test 3: Huge browser_steps array\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\n            \"url\": test_url,\n            \"browser_steps\": [\n                {\"operation\": \"click\", \"selector\": \"#test\" * 1000, \"optional_value\": \"\"}\n            ] * 100\n        }),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    # Should reject or limit\n    assert res.status_code in [201, 400, 413]\n    if res.status_code == 201:\n        watch_uuid = res.json.get('uuid')\n        client.delete(url_for(\"watch\", uuid=watch_uuid), headers={'x-api-key': api_key})\n\n    # Test 4: Extremely long title\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\n            \"url\": test_url,\n            \"title\": \"x\" * 100000  # 100KB title\n        }),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    # Should reject (exceeds maxLength: 5000)\n    assert res.status_code == 400\n\n    delete_all_watches(client)\n\n\ndef test_api_utf8_encoding_edge_cases(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test UTF-8 encoding edge cases that have caused bugs on Windows.\n    Addresses 18+ encoding bugs from changelog.\n    \"\"\"\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n    set_original_response(datastore_path=datastore_path)\n    test_url = url_for('test_endpoint', _external=True)\n\n    # Test 1: Unicode in title (should work)\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\n            \"url\": test_url,\n            \"title\": \"Test 中文 Ελληνικά 日本語 🔥\"\n        }),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    assert res.status_code == 201\n    watch_uuid = res.json.get('uuid')\n\n    # Verify it round-trips correctly\n    res = client.get(url_for(\"watch\", uuid=watch_uuid), headers={'x-api-key': api_key})\n    assert res.status_code == 200\n    assert \"中文\" in res.json.get('title')\n\n    client.delete(url_for(\"watch\", uuid=watch_uuid), headers={'x-api-key': api_key})\n\n    # Test 2: Unicode in URL query parameters\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\n            \"url\": test_url + \"?search=日本語\"\n        }),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    # Should handle URL encoding properly\n    assert res.status_code in [201, 400]\n    if res.status_code == 201:\n        watch_uuid = res.json.get('uuid')\n        client.delete(url_for(\"watch\", uuid=watch_uuid), headers={'x-api-key': api_key})\n\n    # Test 3: Null byte in title (should be rejected or sanitized)\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\n            \"url\": test_url,\n            \"title\": \"Test\\x00Title\"\n        }),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    # Should handle gracefully\n    assert res.status_code in [201, 400]\n    if res.status_code == 201:\n        watch_uuid = res.json.get('uuid')\n        client.delete(url_for(\"watch\", uuid=watch_uuid), headers={'x-api-key': api_key})\n\n    # Test 4: BOM (Byte Order Mark) in title\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\n            \"url\": test_url,\n            \"title\": \"\\ufeffTest with BOM\"\n        }),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    assert res.status_code in [201, 400]\n    if res.status_code == 201:\n        watch_uuid = res.json.get('uuid')\n        client.delete(url_for(\"watch\", uuid=watch_uuid), headers={'x-api-key': api_key})\n\n    delete_all_watches(client)\n\n\ndef test_api_concurrency_race_conditions(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test concurrent API requests to detect race conditions.\n    Addresses 20+ concurrency bugs from changelog.\n    \"\"\"\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n    set_original_response(datastore_path=datastore_path)\n    test_url = url_for('test_endpoint', _external=True)\n\n    # Create a watch\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\"url\": test_url, \"title\": \"Concurrency test\"}),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    assert res.status_code == 201\n    watch_uuid = res.json.get('uuid')\n    wait_for_all_checks(client)\n\n    # Test 1: Concurrent updates to same watch\n    # Note: Flask test client is not thread-safe, so we test sequential updates instead\n    # Real concurrency issues would be caught in integration tests with actual HTTP requests\n    results = []\n    for i in range(10):\n        try:\n            r = client.put(\n                url_for(\"watch\", uuid=watch_uuid),\n                data=json.dumps({\"title\": f\"Title {i}\"}),\n                headers={'content-type': 'application/json', 'x-api-key': api_key},\n            )\n            results.append(r.status_code)\n        except Exception as e:\n            results.append(str(e))\n\n    # All updates should succeed (200) without crashes\n    assert all(r == 200 for r in results), f\"Some updates failed: {results}\"\n\n    # Test 2: Update while watch is being checked\n    # Queue a recheck\n    client.get(\n        url_for(\"watch\", uuid=watch_uuid, recheck=True),\n        headers={'x-api-key': api_key}\n    )\n\n    # Immediately update it\n    res = client.put(\n        url_for(\"watch\", uuid=watch_uuid),\n        data=json.dumps({\"title\": \"Updated during check\"}),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    # Should succeed without error\n    assert res.status_code == 200\n\n    # Test 3: Delete watch that's being processed\n    # Create another watch\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\"url\": test_url}),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    watch_uuid2 = res.json.get('uuid')\n\n    # Queue it for checking\n    client.get(url_for(\"watch\", uuid=watch_uuid2, recheck=True), headers={'x-api-key': api_key})\n\n    # Immediately delete it\n    res = client.delete(url_for(\"watch\", uuid=watch_uuid2), headers={'x-api-key': api_key})\n    # Should succeed or return appropriate error\n    assert res.status_code in [204, 404, 400]\n\n    # Cleanup\n    client.delete(url_for(\"watch\", uuid=watch_uuid), headers={'x-api-key': api_key})\n    delete_all_watches(client)\n\n\n# ============================================================================\n# TIER 2: IMPORTANT FUNCTIONALITY TESTS\n# ============================================================================\n\ndef test_api_time_validation_edge_cases(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test time_between_check validation edge cases.\n    \"\"\"\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n    set_original_response(datastore_path=datastore_path)\n    test_url = url_for('test_endpoint', _external=True)\n\n    # Test 1: Zero interval\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\n            \"url\": test_url,\n            \"time_between_check_use_default\": False,\n            \"time_between_check\": {\"seconds\": 0}\n        }),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    assert res.status_code == 400, \"Zero interval should be rejected\"\n\n    # Test 2: Negative interval\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\n            \"url\": test_url,\n            \"time_between_check_use_default\": False,\n            \"time_between_check\": {\"seconds\": -100}\n        }),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    assert res.status_code == 400, \"Negative interval should be rejected\"\n\n    # Test 3: All fields null with use_default=false\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\n            \"url\": test_url,\n            \"time_between_check_use_default\": False,\n            \"time_between_check\": {\"weeks\": None, \"days\": None, \"hours\": None, \"minutes\": None, \"seconds\": None}\n        }),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    assert res.status_code == 400, \"All null intervals should be rejected when not using default\"\n\n    # Test 4: Extremely large interval (overflow risk)\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\n            \"url\": test_url,\n            \"time_between_check_use_default\": False,\n            \"time_between_check\": {\"weeks\": 999999999}\n        }),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    # Should either accept (with limits) or reject\n    assert res.status_code in [201, 400]\n    if res.status_code == 201:\n        watch_uuid = res.json.get('uuid')\n        client.delete(url_for(\"watch\", uuid=watch_uuid), headers={'x-api-key': api_key})\n\n    # Test 5: Valid minimal interval (should work)\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\n            \"url\": test_url,\n            \"time_between_check_use_default\": False,\n            \"time_between_check\": {\"seconds\": 60}\n        }),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    assert res.status_code == 201\n    watch_uuid = res.json.get('uuid')\n    client.delete(url_for(\"watch\", uuid=watch_uuid), headers={'x-api-key': api_key})\n\n    delete_all_watches(client)\n\n\ndef test_api_browser_steps_validation(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test browser_steps validation for invalid operations and structures.\n    \"\"\"\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n    set_original_response(datastore_path=datastore_path)\n    test_url = url_for('test_endpoint', _external=True)\n\n    # Test 1: Empty browser step\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\n            \"url\": test_url,\n            \"browser_steps\": [\n                {\"operation\": \"\", \"selector\": \"\", \"optional_value\": \"\"}\n            ]\n        }),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    # Should accept (empty is valid as null)\n    assert res.status_code in [201, 400]\n    if res.status_code == 201:\n        watch_uuid = res.json.get('uuid')\n        client.delete(url_for(\"watch\", uuid=watch_uuid), headers={'x-api-key': api_key})\n\n    # Test 2: Invalid operation type\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\n            \"url\": test_url,\n            \"browser_steps\": [\n                {\"operation\": \"invalid_operation\", \"selector\": \"#test\", \"optional_value\": \"\"}\n            ]\n        }),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    # Should accept (validation happens at runtime) or reject\n    assert res.status_code in [201, 400]\n    if res.status_code == 201:\n        watch_uuid = res.json.get('uuid')\n        client.delete(url_for(\"watch\", uuid=watch_uuid), headers={'x-api-key': api_key})\n\n    # Test 3: Missing required fields in browser step\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\n            \"url\": test_url,\n            \"browser_steps\": [\n                {\"operation\": \"click\"}  # Missing selector and optional_value\n            ]\n        }),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    # Should be rejected due to schema validation\n    assert res.status_code == 400\n\n    # Test 4: Extra fields in browser step\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\n            \"url\": test_url,\n            \"browser_steps\": [\n                {\"operation\": \"click\", \"selector\": \"#test\", \"optional_value\": \"\", \"extra_field\": \"value\"}\n            ]\n        }),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    # Should be rejected due to additionalProperties: false\n    assert res.status_code == 400\n\n    delete_all_watches(client)\n\n\ndef test_api_queue_manipulation(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test queue behavior under stress and edge cases.\n    \"\"\"\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n    set_original_response(datastore_path=datastore_path)\n    test_url = url_for('test_endpoint', _external=True)\n\n    # Test 1: Create many watches rapidly\n    watch_uuids = []\n    for i in range(20):\n        res = client.post(\n            url_for(\"createwatch\"),\n            data=json.dumps({\"url\": test_url, \"title\": f\"Watch {i}\"}),\n            headers={'content-type': 'application/json', 'x-api-key': api_key},\n        )\n        if res.status_code == 201:\n            watch_uuids.append(res.json.get('uuid'))\n\n    assert len(watch_uuids) == 20, \"Should be able to create 20 watches\"\n\n    # Test 2: Recheck all when watches exist\n    res = client.get(\n        url_for(\"createwatch\", recheck_all='1'),\n        headers={'x-api-key': api_key},\n    )\n    # Should return success (200 or 202 for background processing)\n    assert res.status_code in [200, 202]\n\n    # Test 3: Verify queue doesn't overflow with moderate load\n    # The app has MAX_QUEUE_SIZE = 5000, we're well below that\n    wait_for_all_checks(client)\n\n    # Cleanup\n    for uuid in watch_uuids:\n        client.delete(url_for(\"watch\", uuid=uuid), headers={'x-api-key': api_key})\n\n    delete_all_watches(client)\n\n\n# ============================================================================\n# TIER 3: EDGE CASES & POLISH\n# ============================================================================\n\ndef test_api_history_edge_cases(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test history API with invalid timestamps and edge cases.\n    \"\"\"\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n    set_original_response(datastore_path=datastore_path)\n    test_url = url_for('test_endpoint', _external=True)\n\n    # Create watch and generate history\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\"url\": test_url}),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    watch_uuid = res.json.get('uuid')\n    wait_for_all_checks(client)\n\n    # Test 1: Get history with invalid timestamp\n    res = client.get(\n        url_for(\"watchsinglehistory\", uuid=watch_uuid, timestamp=\"invalid\"),\n        headers={'x-api-key': api_key}\n    )\n    assert res.status_code == 404, \"Invalid timestamp should return 404\"\n\n    # Test 2: Future timestamp\n    res = client.get(\n        url_for(\"watchsinglehistory\", uuid=watch_uuid, timestamp=\"9999999999\"),\n        headers={'x-api-key': api_key}\n    )\n    assert res.status_code == 404, \"Future timestamp should return 404\"\n\n    # Test 3: Negative timestamp\n    res = client.get(\n        url_for(\"watchsinglehistory\", uuid=watch_uuid, timestamp=\"-1\"),\n        headers={'x-api-key': api_key}\n    )\n    assert res.status_code == 404, \"Negative timestamp should return 404\"\n\n    # Test 4: Diff with reversed timestamps (from > to)\n    # First get actual timestamps\n    res = client.get(\n        url_for(\"watchhistory\", uuid=watch_uuid),\n        headers={'x-api-key': api_key}\n    )\n    if len(res.json) >= 2:\n        timestamps = sorted(res.json.keys())\n        # Try reversed order\n        res = client.get(\n            url_for(\"watchhistorydiff\", uuid=watch_uuid, from_timestamp=timestamps[-1], to_timestamp=timestamps[0]),\n            headers={'x-api-key': api_key}\n        )\n        # Should either work (show reverse diff) or return error\n        assert res.status_code in [200, 400]\n\n    # Cleanup\n    client.delete(url_for(\"watch\", uuid=watch_uuid), headers={'x-api-key': api_key})\n    delete_all_watches(client)\n\n\ndef test_api_notification_edge_cases(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test notification configuration edge cases.\n    \"\"\"\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n    set_original_response(datastore_path=datastore_path)\n    test_url = url_for('test_endpoint', _external=True)\n\n    # Test 1: Invalid notification URL\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\n            \"url\": test_url,\n            \"notification_urls\": [\"invalid://url\", \"ftp://test.com\"]\n        }),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    # Should accept (apprise validates at runtime) or reject\n    assert res.status_code in [201, 400]\n    if res.status_code == 201:\n        watch_uuid = res.json.get('uuid')\n        client.delete(url_for(\"watch\", uuid=watch_uuid), headers={'x-api-key': api_key})\n\n    # Test 2: Invalid notification format\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\n            \"url\": test_url,\n            \"notification_format\": \"invalid_format\"\n        }),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    # Should be rejected by schema\n    assert res.status_code == 400\n\n    # Test 3: Empty notification arrays\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\n            \"url\": test_url,\n            \"notification_urls\": []\n        }),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    # Should accept (empty is valid)\n    assert res.status_code == 201\n    watch_uuid = res.json.get('uuid')\n    client.delete(url_for(\"watch\", uuid=watch_uuid), headers={'x-api-key': api_key})\n\n    delete_all_watches(client)\n\n\ndef test_api_tag_edge_cases(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test tag/group API edge cases including XSS and path traversal.\n    \"\"\"\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n\n    # Test 1: Empty tag title\n    res = client.post(\n        url_for(\"tag\"),\n        data=json.dumps({\"title\": \"\"}),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    # Should be rejected (empty title)\n    assert res.status_code == 400\n\n    # Test 2: XSS in tag title\n    res = client.post(\n        url_for(\"tag\"),\n        data=json.dumps({\"title\": \"<script>alert(1)</script>\"}),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    # Should accept but sanitize\n    if res.status_code == 201:\n        tag_uuid = res.json.get('uuid')\n        # Verify title is stored safely\n        res = client.get(url_for(\"tag\", uuid=tag_uuid), headers={'x-api-key': api_key})\n        # Should be escaped or sanitized\n        client.delete(url_for(\"tag\", uuid=tag_uuid), headers={'x-api-key': api_key})\n\n    # Test 3: Path traversal in tag title\n    res = client.post(\n        url_for(\"tag\"),\n        data=json.dumps({\"title\": \"../../etc/passwd\"}),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    # Should accept (it's just a string, not a path)\n    if res.status_code == 201:\n        tag_uuid = res.json.get('uuid')\n        client.delete(url_for(\"tag\", uuid=tag_uuid), headers={'x-api-key': api_key})\n\n    # Test 4: Very long tag title\n    res = client.post(\n        url_for(\"tag\"),\n        data=json.dumps({\"title\": \"x\" * 10000}),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n    )\n    # Should be rejected (exceeds maxLength)\n    assert res.status_code == 400\n\n\ndef test_api_authentication_edge_cases(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test API authentication edge cases.\n    \"\"\"\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n    set_original_response(datastore_path=datastore_path)\n    test_url = url_for('test_endpoint', _external=True)\n\n    # Test 1: Missing API key\n    res = client.get(url_for(\"createwatch\"))\n    assert res.status_code == 403, \"Missing API key should be forbidden\"\n\n    # Test 2: Invalid API key\n    res = client.get(\n        url_for(\"createwatch\"),\n        headers={'x-api-key': \"invalid_key_12345\"}\n    )\n    assert res.status_code == 403, \"Invalid API key should be forbidden\"\n\n    # Test 3: API key with special characters\n    res = client.get(\n        url_for(\"createwatch\"),\n        headers={'x-api-key': \"key<script>alert(1)</script>\"}\n    )\n    assert res.status_code == 403, \"Invalid API key should be forbidden\"\n\n    # Test 4: Very long API key\n    res = client.get(\n        url_for(\"createwatch\"),\n        headers={'x-api-key': \"x\" * 10000}\n    )\n    assert res.status_code == 403, \"Invalid API key should be forbidden\"\n\n    # Test 5: Case sensitivity of API key\n    wrong_case_key = api_key.upper() if api_key.islower() else api_key.lower()\n    res = client.get(\n        url_for(\"createwatch\"),\n        headers={'x-api-key': wrong_case_key}\n    )\n    # Should be forbidden (keys are case-sensitive)\n    assert res.status_code == 403, \"Wrong case API key should be forbidden\"\n\n    # Test 6: Valid API key should work\n    res = client.get(\n        url_for(\"createwatch\"),\n        headers={'x-api-key': api_key}\n    )\n    assert res.status_code == 200, \"Valid API key should work\"\n"
  },
  {
    "path": "changedetectionio/tests/test_api_tags.py",
    "content": "#!/usr/bin/env python3\n\nfrom flask import url_for\nfrom .util import live_server_setup, wait_for_all_checks, set_original_response\nimport json\nimport time\n\ndef test_api_tags_listing(client, live_server, measure_memory_usage, datastore_path):\n   #  live_server_setup(live_server) # Setup on conftest per function\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n    tag_title = 'Test Tag'\n\n\n    set_original_response(datastore_path=datastore_path)\n\n\n    res = client.get(\n        url_for(\"tags\"),\n        headers={'x-api-key': api_key}\n    )\n    assert res.get_data(as_text=True).strip() == \"{}\", \"Should be empty list\"\n    assert res.status_code == 200\n\n    res = client.post(\n        url_for(\"tag\"),\n        data=json.dumps({\"title\": tag_title}),\n        headers={'content-type': 'application/json', 'x-api-key': api_key}\n    )\n    assert res.status_code == 201\n\n    new_tag_uuid = res.json.get('uuid')\n\n    # List tags - should include our new tag\n    res = client.get(\n        url_for(\"tags\"),\n        headers={'x-api-key': api_key}\n    )\n    assert res.status_code == 200\n    assert new_tag_uuid in res.get_data(as_text=True)\n    assert res.json[new_tag_uuid]['title'] == tag_title\n    assert res.json[new_tag_uuid]['notification_muted'] == False\n\n    # Get single tag\n    res = client.get(\n        url_for(\"tag\", uuid=new_tag_uuid),\n        headers={'x-api-key': api_key}\n    )\n    assert res.status_code == 200\n    assert res.json['title'] == tag_title\n\n    # Update tag\n    res = client.put(\n        url_for(\"tag\", uuid=new_tag_uuid),\n        data=json.dumps({\"title\": \"Updated Tag\"}),\n        headers={'content-type': 'application/json', 'x-api-key': api_key}\n    )\n    assert res.status_code == 200\n    assert b'OK' in res.data\n\n    # Verify update worked\n    res = client.get(\n        url_for(\"tag\", uuid=new_tag_uuid),\n        headers={'x-api-key': api_key}\n    )\n    assert res.status_code == 200\n    assert res.json['title'] == 'Updated Tag'\n\n    # Mute tag notifications\n    res = client.get(\n        url_for(\"tag\", uuid=new_tag_uuid) + \"?muted=muted\",\n        headers={'x-api-key': api_key}\n    )\n    assert res.status_code == 200\n    assert b'OK' in res.data\n\n    # Verify muted status\n    res = client.get(\n        url_for(\"tag\", uuid=new_tag_uuid),\n        headers={'x-api-key': api_key}\n    )\n    assert res.status_code == 200\n    assert res.json['notification_muted'] == True\n\n    # Unmute tag\n    res = client.get(\n        url_for(\"tag\", uuid=new_tag_uuid) + \"?muted=unmuted\",\n        headers={'x-api-key': api_key}\n    )\n    assert res.status_code == 200\n    assert b'OK' in res.data\n\n    # Verify unmuted status\n    res = client.get(\n        url_for(\"tag\", uuid=new_tag_uuid),\n        headers={'x-api-key': api_key}\n    )\n    assert res.status_code == 200\n    assert res.json['notification_muted'] == False\n\n    # Create a watch with the tag and check it matches UUID\n    test_url = url_for('test_endpoint', _external=True)\n    res = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\"url\": test_url, \"tag\": \"Updated Tag\", \"title\": \"Watch with tag\"}),\n        headers={'content-type': 'application/json', 'x-api-key': api_key},\n        follow_redirects=True\n    )\n    assert res.status_code == 201\n    watch_uuid = res.json.get('uuid')\n\n\n    wait_for_all_checks()\n    # Verify tag is associated with watch by name if need be\n    res = client.get(\n        url_for(\"watch\", uuid=watch_uuid),\n        headers={'x-api-key': api_key}\n    )\n    assert res.status_code == 200\n    assert new_tag_uuid in res.json.get('tags', [])\n\n    # Test that tags are returned when listing ALL watches (issue #3854)\n    res = client.get(\n        url_for(\"createwatch\"),  # GET /api/v1/watch - list all watches\n        headers={'x-api-key': api_key}\n    )\n    assert res.status_code == 200\n    assert watch_uuid in res.json, \"Watch should be in the list\"\n    assert 'tags' in res.json[watch_uuid], \"Tags field should be present in watch list\"\n    assert new_tag_uuid in res.json[watch_uuid]['tags'], \"Tag UUID should be in tags array\"\n\n    # Check recheck by tag\n    before_check_time = live_server.app.config['DATASTORE'].data['watching'][watch_uuid].get('last_checked')\n    time.sleep(1)\n    res = client.get(\n       url_for(\"tag\", uuid=new_tag_uuid) + \"?recheck=true\",\n       headers={'x-api-key': api_key}\n    )\n\n    assert res.status_code == 200\n    assert b'OK, queued 1 watches for rechecking' in res.data\n\n\n    wait_for_all_checks()\n    after_check_time = live_server.app.config['DATASTORE'].data['watching'][watch_uuid].get('last_checked')\n\n    assert before_check_time != after_check_time\n\n    # Delete tag\n    res = client.delete(\n        url_for(\"tag\", uuid=new_tag_uuid),\n        headers={'x-api-key': api_key}\n    )\n    assert res.status_code == 204\n\n    # Verify tag is gone\n    res = client.get(\n        url_for(\"tags\"),\n        headers={'x-api-key': api_key}\n    )\n    assert res.status_code == 200\n    assert new_tag_uuid not in res.get_data(as_text=True)\n\n    # Verify tag was removed from watch\n    res = client.get(\n        url_for(\"watch\", uuid=watch_uuid),\n        headers={'x-api-key': api_key}\n    )\n    assert res.status_code == 200\n    assert new_tag_uuid not in res.json.get('tags', [])\n\n    # Delete the watch\n    res = client.delete(\n        url_for(\"watch\", uuid=watch_uuid),\n        headers={'x-api-key': api_key},\n    )\n    assert res.status_code == 204\n\n\ndef test_api_tag_restock_processor_config(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test that a tag/group can be created and updated with processor_config_restock_diff via the API.\n    Since Tag extends WatchBase, processor config fields injected into WatchBase are also valid for tags.\n    \"\"\"\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n\n    set_original_response(datastore_path=datastore_path)\n\n    # Create a tag with processor_config_restock_diff in a single POST (issue #3966)\n    res = client.post(\n        url_for(\"tag\"),\n        data=json.dumps({\n            \"title\": \"Restock Group\",\n            \"overrides_watch\": True,\n            \"processor_config_restock_diff\": {\n                \"in_stock_processing\": \"in_stock_only\",\n                \"follow_price_changes\": True,\n                \"price_change_min\": 7777777\n            }\n        }),\n        headers={'content-type': 'application/json', 'x-api-key': api_key}\n    )\n    assert res.status_code == 201, f\"POST tag with restock config failed: {res.data}\"\n    tag_uuid = res.json.get('uuid')\n\n    # Verify processor config was saved during creation (the bug: these were discarded)\n    res = client.get(\n        url_for(\"tag\", uuid=tag_uuid),\n        headers={'x-api-key': api_key}\n    )\n    assert res.status_code == 200\n    tag_data = res.json\n    assert tag_data.get('overrides_watch') == True, \"overrides_watch should be saved on POST\"\n    assert tag_data.get('processor_config_restock_diff', {}).get('in_stock_processing') == 'in_stock_only', \\\n        \"processor_config_restock_diff should be saved on POST\"\n    assert tag_data.get('processor_config_restock_diff', {}).get('price_change_min') == 7777777, \\\n        \"price_change_min should be saved on POST\"\n\n    # Update tag with valid processor_config_restock_diff via PUT\n    res = client.put(\n        url_for(\"tag\", uuid=tag_uuid),\n        headers={'x-api-key': api_key, 'content-type': 'application/json'},\n        data=json.dumps({\n            \"overrides_watch\": True,\n            \"processor_config_restock_diff\": {\n                \"in_stock_processing\": \"in_stock_only\",\n                \"follow_price_changes\": True,\n                \"price_change_min\": 8888888\n            }\n        })\n    )\n    assert res.status_code == 200, f\"PUT tag with restock config failed: {res.data}\"\n\n    # Verify the config was stored via API\n    res = client.get(\n        url_for(\"tag\", uuid=tag_uuid),\n        headers={'x-api-key': api_key}\n    )\n    assert res.status_code == 200\n    tag_data = res.json\n    assert tag_data.get('overrides_watch') == True\n    assert tag_data.get('processor_config_restock_diff', {}).get('in_stock_processing') == 'in_stock_only'\n    assert tag_data.get('processor_config_restock_diff', {}).get('price_change_min') == 8888888\n\n    # Verify the value is also reflected in the UI tag edit page\n    res = client.get(url_for(\"tags.form_tag_edit\", uuid=tag_uuid))\n    assert res.status_code == 200\n    assert b'8888888' in res.data, \"price_change_min set via API should appear in the UI tag edit form\"\n\n    # Invalid enum value should be rejected by OpenAPI spec validation\n    res = client.put(\n        url_for(\"tag\", uuid=tag_uuid),\n        headers={'x-api-key': api_key, 'content-type': 'application/json'},\n        data=json.dumps({\n            \"processor_config_restock_diff\": {\n                \"in_stock_processing\": \"not_a_valid_value\"\n            }\n        })\n    )\n    assert res.status_code == 400\n    assert b'Validation failed' in res.data\n\n    # Clean up\n    res = client.delete(\n        url_for(\"tag\", uuid=tag_uuid),\n        headers={'x-api-key': api_key}\n    )\n    assert res.status_code == 204\n\n\ndef test_roundtrip_API(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test the full round trip, this way we test the default Model fits back into OpenAPI spec\n    :param client:\n    :param live_server:\n    :param measure_memory_usage:\n    :param datastore_path:\n    :return:\n    \"\"\"\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n\n    set_original_response(datastore_path=datastore_path)\n\n    res = client.post(\n        url_for(\"tag\"),\n        data=json.dumps({\"title\": \"My tag title\"}),\n        headers={'content-type': 'application/json', 'x-api-key': api_key}\n    )\n    assert res.status_code == 201\n\n    uuid = res.json.get('uuid')\n\n    # Now fetch it and send it back\n\n    res = client.get(\n        url_for(\"tag\", uuid=uuid),\n        headers={'x-api-key': api_key}\n    )\n\n    tag = res.json\n\n    # Only test with date_created (readOnly field that should be filtered out)\n    # last_changed is Watch-specific and doesn't apply to Tags\n    tag['date_created'] = 454444444444\n\n    # HTTP PUT ( UPDATE an existing watch )\n    res = client.put(\n        url_for(\"tag\", uuid=uuid),\n        headers={'x-api-key': api_key, 'content-type': 'application/json'},\n        data=json.dumps(tag),\n    )\n    if res.status_code != 200:\n        print(f\"\\n=== PUT failed with {res.status_code} ===\")\n        print(f\"Error: {res.data}\")\n    assert res.status_code == 200, \"HTTP PUT update was sent OK\"\n\n    # Verify readOnly fields like date_created cannot be overridden\n    res = client.get(\n        url_for(\"tag\", uuid=uuid),\n        headers={'x-api-key': api_key}\n    )\n    date_created = res.json.get('date_created')\n    assert date_created != 454444444444, \"ReadOnly date_created should not be updateable\"\n    assert date_created != \"454444444444\", \"ReadOnly date_created should not be updateable\"\n"
  },
  {
    "path": "changedetectionio/tests/test_auth.py",
    "content": "#!/usr/bin/env python3\n\nimport time\nfrom flask import url_for\nfrom .util import live_server_setup, wait_for_all_checks\n\n# test pages with http://username@password:foobar.com/ work\ndef test_basic_auth(client, live_server, measure_memory_usage, datastore_path):\n   #  live_server_setup(live_server) # Setup on conftest per function\n\n\n    # This page will echo back any auth info\n    test_url = url_for('test_basicauth_method', _external=True).replace(\"//\",\"//myuser:mypass@\")\n    time.sleep(1)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n    time.sleep(1)\n    # Check form validation\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={\"include_filters\": \"\", \"url\": test_url, \"tags\": \"\", \"headers\": \"\", 'fetch_backend': \"html_requests\", \"time_between_check_use_default\": \"y\"},\n        follow_redirects=True\n    )\n    assert b\"Updated watch.\" in res.data\n\n    wait_for_all_checks(client)\n    res = client.get(\n        url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"),\n        follow_redirects=True\n    )\n\n    assert b'myuser mypass basic' in res.data\n"
  },
  {
    "path": "changedetectionio/tests/test_automatic_follow_ldjson_price.py",
    "content": "#!/usr/bin/env python3\n\nimport time\nfrom flask import url_for\nfrom .util import live_server_setup, extract_UUID_from_client, wait_for_all_checks, delete_all_watches\nimport os\n\n\ndef set_response_with_ldjson(datastore_path):\n    test_return_data = \"\"\"<html>\n       <body>\n     Some initial text<br>\n     <p>Which is across multiple lines</p>\n     <br>\n     So let's see what happens.  <br>\n     <div class=\"sametext\">Some text thats the same</div>\n     <div class=\"changetext\">Some text that will change</div>\n     <script type=\"application/ld+json\">\n        {\n           \"@context\":\"https://schema.org/\",\n           \"@type\":\"Product\",\n           \"@id\":\"https://www.some-virtual-phone-shop.com/celular-iphone-14/p\",\n           \"name\":\"Celular Iphone 14 Pro Max 256Gb E Sim A16 Bionic\",\n           \"brand\":{\n              \"@type\":\"Brand\",\n              \"name\":\"APPLE\"\n           },\n           \"image\":\"https://www.some-virtual-phone-shop.com/15509426/image.jpg\",\n           \"description\":\"You dont need it\",\n           \"mpn\":\"111111\",\n           \"sku\":\"22222\",\n           \"Offers\":{\n              \"@type\":\"AggregateOffer\",\n              \"lowPrice\":8097000,\n              \"highPrice\":8099900,\n              \"priceCurrency\":\"COP\",\n              \"offers\":[\n                 {\n                    \"@type\":\"Offer\",\n                    \"price\":8097000,\n                    \"priceCurrency\":\"COP\",\n                    \"availability\":\"http://schema.org/InStock\",\n                    \"sku\":\"102375961\",\n                    \"itemCondition\":\"http://schema.org/NewCondition\",\n                    \"seller\":{\n                       \"@type\":\"Organization\",\n                       \"name\":\"ajax\"\n                    }\n                 }\n              ],\n              \"offerCount\":1\n           }\n        }\n       </script>\n     </body>\n     </html>\n\"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n    return None\n\ndef set_response_without_ldjson(datastore_path):\n    test_return_data = \"\"\"<html>\n       <body>\n     Some initial text<br>\n     <p>Which is across multiple lines</p>\n     <br>\n     So let's see what happens.  <br>\n     <div class=\"sametext\">Some text thats the same</div>\n     <div class=\"changetext\">Some text that will change</div>     \n     </body>\n     </html>\n\"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n    return None\n\n# def test_setup(client, live_server, measure_memory_usage, datastore_path):\n   #  live_server_setup(live_server) # Setup on conftest per function\n\n# actually only really used by the distll.io importer, but could be handy too\ndef test_check_ldjson_price_autodetect(client, live_server, measure_memory_usage, datastore_path):\n    \n    set_response_with_ldjson(datastore_path=datastore_path)\n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    # Should get a notice that it's available\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'ldjson-price-track-offer' in res.data\n\n    # Accept it\n    client.get(url_for('price_data_follower.accept', uuid=uuid, follow_redirects=True))\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n    # Offer should be gone\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'Embedded price data' not in res.data\n    assert b'processor-badge-restock_diff' in res.data\n\n    # and last snapshop (via API) should be just the price\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n    res = client.get(\n        url_for(\"watchsinglehistory\", uuid=uuid, timestamp='latest'),\n        headers={'x-api-key': api_key},\n    )\n\n    assert b'8097000' in res.data\n\n    # And not this cause its not the ld-json\n    assert b\"So let's see what happens\" not in res.data\n\n    delete_all_watches(client)\n\n    ##########################################################################################\n    # And we shouldnt see the offer\n    set_response_without_ldjson(datastore_path=datastore_path)\n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'ldjson-price-track-offer' not in res.data\n    \n    ##########################################################################################\n    delete_all_watches(client)\n\n\ndef _test_runner_check_bad_format_ignored(live_server, client, has_ldjson_price_data):\n\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    for k,v in client.application.config.get('DATASTORE').data['watching'].items():\n        assert v.get('last_error') == False\n        assert v.get('has_ldjson_price_data') == has_ldjson_price_data, f\"Detected LDJSON data? should be {has_ldjson_price_data}\"\n\n\n    ##########################################################################################\n    delete_all_watches(client)\n\n\ndef test_bad_ldjson_is_correctly_ignored(client, live_server, measure_memory_usage, datastore_path):\n    \n    test_return_data = \"\"\"\n            <html>\n            <head>\n                <script type=\"application/ld+json\">\n                    {\n                        \"@context\": \"http://schema.org\",\n                        \"@type\": [\"Product\", \"SubType\"],\n                        \"name\": \"My test product\",\n                        \"description\": \"\",\n                        \"offers\": {\n                            \"note\" : \"You can see the case-insensitive OffERS key, it should work\",\n                            \"@type\": \"Offer\",\n                            \"offeredBy\": {\n                                \"@type\": \"Organization\",\n                                \"name\":\"Person\",\n                                \"telephone\":\"+1 999 999 999\"\n                            },\n                            \"price\": \"1\",\n                            \"priceCurrency\": \"EUR\",\n                            \"url\": \"/some/url\"\n                        }\n                    }\n                </script>\n            </head>\n            <body>\n            <div class=\"yes\">Some extra stuff</div>\n            </body></html>\n     \"\"\"\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n\n    _test_runner_check_bad_format_ignored(live_server=live_server, client=client, has_ldjson_price_data=True)\n\n    # This is OK that it offers a suggestion in this case, the processor will let them know more about something wrong\n\n    # test_return_data = \"\"\"\n    #         <html>\n    #         <head>\n    #             <script type=\"application/ld+json\">\n    #                 {\n    #                     \"@context\": \"http://schema.org\",\n    #                     \"@type\": [\"Product\", \"SubType\"],\n    #                     \"name\": \"My test product\",\n    #                     \"description\": \"\",\n    #                     \"BrokenOffers\": {\n    #                         \"@type\": \"Offer\",\n    #                         \"offeredBy\": {\n    #                             \"@type\": \"Organization\",\n    #                             \"name\":\"Person\",\n    #                             \"telephone\":\"+1 999 999 999\"\n    #                         },\n    #                         \"price\": \"1\",\n    #                         \"priceCurrency\": \"EUR\",\n    #                         \"url\": \"/some/url\"\n    #                     }\n    #                 }\n    #             </script>\n    #         </head>\n    #         <body>\n    #         <div class=\"yes\">Some extra stuff</div>\n    #         </body></html>\n    #  \"\"\"\n    # with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n    #     f.write(test_return_data)\n    #\n    # _test_runner_check_bad_format_ignored(live_server=live_server, client=client, has_ldjson_price_data=False)\n"
  },
  {
    "path": "changedetectionio/tests/test_backend.py",
    "content": "#!/usr/bin/env python3\nimport os\n\nimport time\nfrom flask import url_for\nfrom .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, extract_rss_token_from_UI, \\\n    extract_UUID_from_client, delete_all_watches\n\n\n# Basic test to check inscriptus is not adding return line chars, basically works etc\ndef test_inscriptus():\n    from inscriptis import get_text\n    html_content = \"<html><body>test!<br>ok man</body></html>\"\n    stripped_text_from_html = get_text(html_content)\n    assert stripped_text_from_html == 'test!\\nok man'\n\n\n\ndef test_check_basic_change_detection_functionality(client, live_server, measure_memory_usage, datastore_path):\n    set_original_response(datastore_path=datastore_path)\n\n    uuid = client.application.config.get('DATASTORE').add_watch(url=url_for('test_endpoint', _external=True))\n\n    # Do this a few times.. ensures we dont accidently set the status\n    for n in range(3):\n        client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n        # Give the thread time to pick it up\n        wait_for_all_checks(client)\n\n        # It should report nothing found (no new 'has-unread-changes' class)\n        res = client.get(url_for(\"watchlist.index\"))\n        assert b'has-unread-changes' not in res.data\n        assert b'test-endpoint' in res.data\n\n        # Default no password set, this stuff should be always available.\n\n        assert b\"SETTINGS\" in res.data\n        assert b\"IMPORT\" in res.data\n\n    #####################\n\n    # Check HTML conversion detected and workd\n    res = client.get(\n        url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"),\n        follow_redirects=True\n    )\n    # Check this class does not appear (that we didnt see the actual source)\n    assert b'foobar-detection' not in res.data\n\n    # Check POST preview\n    res = client.post(\n        url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"),\n        follow_redirects=True\n    )\n    # Check this class does not appear (that we didnt see the actual source)\n    assert b'foobar-detection' not in res.data\n\n\n    # Make a change\n    set_modified_response(datastore_path=datastore_path)\n\n    # Force recheck\n    res = client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    assert b'Queued 1 watch for rechecking.' in res.data\n\n    wait_for_all_checks(client)\n\n    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))\n\n    # Check the 'get latest snapshot works'\n    res = client.get(url_for(\"ui.ui_edit.watch_get_latest_html\", uuid=uuid))\n    assert b'which has this one new line' in res.data\n\n    # Now something should be ready, indicated by having a 'has-unread-changes' class\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'has-unread-changes' in res.data\n\n    # #75, and it should be in the RSS feed\n    rss_token = extract_rss_token_from_UI(client)\n    res = client.get(url_for(\"rss.feed\", token=rss_token, _external=True))\n    expected_url = url_for('test_endpoint', _external=True)\n    assert b'<rss' in res.data\n\n    # re #16 should have the diff in here too\n    assert b'which has this one new line' in res.data\n    assert b'CDATA' in res.data\n\n#\n    # Following the 'diff' link, it should no longer display as 'has-unread-changes' even after we recheck it a few times\n    res = client.get(url_for(\"ui.ui_diff.diff_history_page\", uuid=uuid))\n    assert b'selected=\"\"' in res.data, \"Confirm diff history page loaded\"\n\n    assert b'Which is across multiple lines' in res.data\n    # The linefeed should have been added ( @BR@ was replaced with a linefeed because this is htmlcolor kinda display )\n    assert b'Which is across multiple lines</span>\\n' in res.data\n\n    # Check the [preview] pulls the right one\n    res = client.get(\n        url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"),\n        follow_redirects=True\n    )\n    assert b'which has this one new line' in res.data\n    assert b'Which is across multiple lines' not in res.data\n\n    wait_for_all_checks(client)\n\n\n    # Do this a few times.. ensures we don't accidently set the status\n    for n in range(2):\n        res = client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n        # Give the thread time to pick it up\n        wait_for_all_checks(client)\n\n        # It should report nothing found (no new 'has-unread-changes' class)\n        res = client.get(url_for(\"watchlist.index\"))\n\n        assert b'has-unread-changes' not in res.data\n        assert b'class=\"has-unread-changes' not in res.data\n        assert b'head title' in res.data  # Should be ON by default\n        assert b'test-endpoint' in res.data\n\n    # Recheck it but only with a title change, content wasnt changed\n    set_original_response(datastore_path=datastore_path, extra_title=\" and more\")\n\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'head title and more' in res.data\n\n    # Be sure the last_viewed is going to be greater than the last snapshot\n    time.sleep(1)\n\n    # hit the mark all viewed link\n    res = client.get(url_for(\"ui.mark_all_viewed\"), follow_redirects=True)\n\n    assert b'class=\"has-unread-changes' not in res.data\n    assert b'has-unread-changes' not in res.data\n\n    # #2458 \"clear history\" should make the Watch object update its status correctly when the first snapshot lands again\n    client.get(url_for(\"ui.clear_watch_history\", uuid=uuid))\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'preview/' in res.data\n\n    #\n    # Cleanup everything\n    delete_all_watches(client)\n\ndef test_title_scraper(client, live_server, measure_memory_usage, datastore_path):\n\n    set_original_response(datastore_path=datastore_path)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=url_for('test_endpoint', _external=True))\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks()\n\n    # It should report nothing found (no new 'has-unread-changes' class)\n    res = client.get(url_for(\"watchlist.index\"))\n\n    assert b'head title' in res.data  # Should be ON by default\n\n    # Recheck it but only with a title change, content wasnt changed\n    set_original_response(datastore_path=datastore_path, extra_title=\" and more\")\n\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'head title and more' in res.data\n\n    # disable <title> pickup\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\"application-ui-use_page_title_in_list\": \"\",\n              \"requests-time_between_check-minutes\": 180,\n              'application-fetch_backend': \"html_requests\"},\n        follow_redirects=True\n    )\n\n    set_original_response(datastore_path=datastore_path, extra_title=\" SHOULD NOT APPEAR\")\n\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'SHOULD NOT APPEAR' not in res.data\n\n    delete_all_watches(client)\n\ndef test_title_scraper_html_only(client, live_server, measure_memory_usage, datastore_path):\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write('\"My text document\\nWhere I talk about <title>\\nwhich should not get registered\\n')\n\n    test_url = url_for('test_endpoint', content_type=\"text/plain\", _external=True)\n\n    uuid = client.application.config.get('DATASTORE').add_watch(test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks()\n\n    # It should report nothing found (no new 'has-unread-changes' class)\n    res = client.get(url_for(\"watchlist.index\"))\n\n    assert b'which should not get registered' not in res.data  # Should be ON by default\n    assert not live_server.app.config['DATASTORE'].data['watching'][uuid].get('title')\n\n\n\n\n# Server says its plaintext, we should always treat it as plaintext, and then if they have a filter, try to apply that\ndef test_requests_timeout(client, live_server, measure_memory_usage, datastore_path):\n    delay = 2\n    test_url = url_for('test_endpoint', delay=delay, _external=True)\n\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\"application-ui-use_page_title_in_list\": \"\",\n              \"requests-time_between_check-minutes\": 180,\n              \"requests-timeout\": delay - 1,\n              'application-fetch_backend': \"html_requests\"},\n        follow_redirects=True\n    )\n\n    # Add our URL to the import page\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    # requests takes >2 sec but we timeout at 1 second\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'Read timed out. (read timeout=1)' in res.data\n\n    ##### Now set a longer timeout\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\"application-ui-use_page_title_in_list\": \"\",\n              \"requests-time_between_check-minutes\": 180,\n              \"requests-timeout\": delay + 1, # timeout should be a second more than the reply time\n              'application-fetch_backend': \"html_requests\"},\n        follow_redirects=True\n    )\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    wait_for_all_checks(client)\n\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'Read timed out' not in res.data\n\ndef test_non_text_mime_or_downloads(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n\n    https://github.com/dgtlmoon/changedetection.io/issues/3434\n    I noticed that a watched website can be monitored fine as long as the server sends content-type: text/plain; charset=utf-8,\n    but once the server sends content-type: application/octet-stream (which is usually done to force the browser to show the Download dialog),\n    changedetection somehow ignores all line breaks and treats the document file as if everything is on one line.\n\n    WHAT THIS DOES - makes the system rely on 'magic' to determine what is it\n\n    :param client:\n    :param live_server:\n    :param measure_memory_usage:\n    :return:\n    \"\"\"\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(\"\"\"some random text that should be split by line\nand not parsed with html_to_text\nthis way we know that it correctly parsed as plain text\n\\r\\n\nok\\r\\n\ngot it\\r\\n\n\"\"\")\n\n    test_url = url_for('test_endpoint', content_type=\"application/octet-stream\", _external=True)\n\n    # Add our URL to the import page\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    wait_for_all_checks(client)\n\n    ### check the front end\n    res = client.get(\n        url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"),\n        follow_redirects=True\n    )\n    assert b\"some random text that should be split by line\\n\" in res.data\n    ####\n\n    # Check the snapshot by API that it has linefeeds too\n    watch_uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n    res = client.get(\n        url_for(\"watchhistory\", uuid=watch_uuid),\n        headers={'x-api-key': api_key},\n    )\n\n    # Fetch a snapshot by timestamp, check the right one was found\n    res = client.get(\n        url_for(\"watchsinglehistory\", uuid=watch_uuid, timestamp=list(res.json.keys())[-1]),\n        headers={'x-api-key': api_key},\n    )\n    assert b\"some random text that should be split by line\\n\" in res.data\n\n\n    delete_all_watches(client)\n\n\ndef test_standard_text_plain(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n\n    https://github.com/dgtlmoon/changedetection.io/issues/3434\n    I noticed that a watched website can be monitored fine as long as the server sends content-type: text/plain; charset=utf-8,\n    but once the server sends content-type: application/octet-stream (which is usually done to force the browser to show the Download dialog),\n    changedetection somehow ignores all line breaks and treats the document file as if everything is on one line.\n\n    The real bug here can be that it will try to process plain-text as HTML, losing \n\n    :param client:\n    :param live_server:\n    :param measure_memory_usage:\n    :return:\n    \"\"\"\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(\"\"\"some random text that should be split by line\nand not parsed with html_to_text\nEven this title should stay because we are just plain text\nthis way we know that it correctly parsed as plain text\n\\r\\n\nok\\r\\n\ngot it\\r\\n\n\"\"\")\n\n    test_url = url_for('test_endpoint', content_type=\"text/plain\", _external=True)\n\n    # Add our URL to the import page\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    wait_for_all_checks(client)\n\n    ### check the front end\n    res = client.get(\n        url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"),\n        follow_redirects=True\n    )\n\n    assert b\"some random text that should be split by line\\n\" in res.data\n    ####\n\n    # Check the snapshot by API that it has linefeeds too\n    watch_uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n    res = client.get(\n        url_for(\"watchhistory\", uuid=watch_uuid),\n        headers={'x-api-key': api_key},\n    )\n\n    # Fetch a snapshot by timestamp, check the right one was found\n    res = client.get(\n        url_for(\"watchsinglehistory\", uuid=watch_uuid, timestamp=list(res.json.keys())[-1]),\n        headers={'x-api-key': api_key},\n    )\n    assert b\"some random text that should be split by line\\n\" in res.data\n    assert b\"Even this title should stay because we are just plain text\" in res.data\n\n    delete_all_watches(client)\n\n# Server says its plaintext, we should always treat it as plaintext\ndef test_plaintext_even_if_xml_content(client, live_server, measure_memory_usage, datastore_path):\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(\"\"\"\n\n    \n    Abonnementen bijwerken\n\n\"\"\")\n\n    test_url = url_for('test_endpoint', content_type=\"text/plain\", _external=True)\n\n    # Add our URL to the import page\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    wait_for_all_checks(client)\n\n    res = client.get(\n        url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"),\n        follow_redirects=True\n    )\n\n    assert b'<string name="feed_update_receiver_name"' in res.data\n\n    delete_all_watches(client)\n\n# Server says its plaintext, we should always treat it as plaintext, and then if they have a filter, try to apply that\ndef test_plaintext_even_if_xml_content_and_can_apply_filters(client, live_server, measure_memory_usage, datastore_path):\n\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(\"\"\"\n\n    \n    Abonnementen bijwerken\n    ok man\n\n\"\"\")\n\n    test_url=url_for('test_endpoint', content_type=\"text/plain\", _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url, extras={\"include_filters\": ['//string']})\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    res = client.get(\n        url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"),\n        follow_redirects=True\n    )\n\n    # Check that the string element with the correct name attribute is present\n    # Note: namespace declarations may be included when extracting elements, which is correct XML behavior\n    assert b'feed_update_receiver_name' in res.data\n    assert b'Abonnementen bijwerken' in res.data\n    assert b'<foobar' not in res.data\n\n    res = delete_all_watches(client)\n"
  },
  {
    "path": "changedetectionio/tests/test_backup.py",
    "content": "#!/usr/bin/env python3\n\nfrom .util import set_original_response, live_server_setup, wait_for_all_checks\nfrom flask import url_for\nimport io\nfrom zipfile import ZipFile, ZIP_DEFLATED\nimport re\nimport time\nfrom changedetectionio.model import Watch, Tag\n\n\ndef test_backup(client, live_server, measure_memory_usage, datastore_path):\n    set_original_response(datastore_path=datastore_path)\n\n\n    # Add our URL to the import page\n    res = client.post(\n        url_for(\"imports.import_page\"),\n        data={\"urls\": url_for('test_endpoint', _external=True)+\"?somechar=őőőőőőőő\"},\n        follow_redirects=True\n    )\n\n    assert b\"1 Imported\" in res.data\n    wait_for_all_checks(client)\n\n    # Launch the thread in the background to create the backup\n    res = client.get(\n        url_for(\"backups.request_backup\"),\n        follow_redirects=True\n    )\n    time.sleep(4)\n\n    res = client.get(\n        url_for(\"backups.create\"),\n        follow_redirects=True\n    )\n    # Can see the download link to the backup\n    assert b'= 2\n\n    backup = ZipFile(io.BytesIO(res.data))\n    l = backup.namelist()\n\n    # Check for UUID-based txt files (history, snapshot, and last-checksum)\n    uuid4hex_txt = re.compile('^[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}.*txt', re.I)\n    txt_files = list(filter(uuid4hex_txt.match, l))\n    # Should be three txt files in the archive (history, snapshot, and last-checksum)\n    assert len(txt_files) == 3\n\n    # Check for watch.json files (new format)\n    uuid4hex_json = re.compile('^[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}/watch\\.json$', re.I)\n    json_files = list(filter(uuid4hex_json.match, l))\n    # Should be one watch.json file in the archive (the imported watch)\n    assert len(json_files) == 1, f\"Expected 1 watch.json file, found {len(json_files)}: {json_files}\"\n\n    # Check for changedetection.json (settings file)\n    assert 'changedetection.json' in l, \"changedetection.json should be in backup\"\n\n    # secret.txt must never be included — it contains the Flask session key\n    assert 'secret.txt' not in l, \"secret.txt (Flask session key) must not be included in backup\"\n\n    # Get the latest one\n    res = client.get(\n        url_for(\"backups.remove_backups\"),\n        follow_redirects=True\n    )\n\n    assert b'No backups found.' in res.data\n\n\ndef test_watch_data_package_download(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"Test downloading a single watch's data as a zip package\"\"\"\n\n    set_original_response(datastore_path=datastore_path)\n\n    uuid = client.application.config.get('DATASTORE').add_watch(url=url_for('test_endpoint', _external=True))\n    tag_uuid = client.application.config.get('DATASTORE').add_tag(title=\"Tasty backup tag\")\n    tag_uuid2 = client.application.config.get('DATASTORE').add_tag(title=\"Tasty backup tag number two\")\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    wait_for_all_checks(client)\n\n    # Download the watch data package\n    res = client.get(url_for(\"ui.ui_edit.watch_get_data_package\", uuid=uuid))\n\n    # Should get the right zip content type\n    assert res.content_type == \"application/zip\"\n\n    # Should be PK/ZIP stream (PKzip header)\n    assert res.data[:2] == b'PK', \"File should start with PK (PKzip header)\"\n    assert res.data.count(b'PK') >= 2, \"Should have multiple PK markers (zip file structure)\"\n\n    # Verify zip contents\n    backup = ZipFile(io.BytesIO(res.data))\n    files = backup.namelist()\n\n    # Should have files in a UUID directory\n    assert any(uuid in f for f in files), f\"Files should be in UUID directory: {files}\"\n\n    # Should contain watch.json\n    watch_json_path = f\"{uuid}/watch.json\"\n    assert watch_json_path in files, f\"Should contain watch.json, got: {files}\"\n\n    # Should contain history/snapshot files\n    uuid4hex_txt = re.compile(f'^{re.escape(uuid)}/.*\\\\.txt', re.I)\n    txt_files = list(filter(uuid4hex_txt.match, files))\n    assert len(txt_files) > 0, f\"Should have at least one .txt file (history/snapshot), got: {files}\"\n\n\ndef test_backup_restore(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"Test that a full backup zip can be restored — watches and tags survive a round-trip.\"\"\"\n\n    set_original_response(datastore_path=datastore_path)\n\n    datastore = live_server.app.config['DATASTORE']\n    watch_url = url_for('test_endpoint', _external=True)\n\n    # Set up: one watch and two tags\n    uuid = datastore.add_watch(url=watch_url)\n    tag_uuid = datastore.add_tag(title=\"Tasty backup tag\")\n    tag_uuid2 = datastore.add_tag(title=\"Tasty backup tag number two\")\n\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    # Create a full backup\n    client.get(url_for(\"backups.request_backup\"), follow_redirects=True)\n    time.sleep(4)\n\n    # Download the latest backup zip\n    res = client.get(url_for(\"backups.download_backup\", filename=\"latest\"), follow_redirects=True)\n    assert res.content_type == \"application/zip\"\n    zip_data = res.data\n\n    # Confirm the zip contains both watch.json and tag.json entries\n    backup = ZipFile(io.BytesIO(zip_data))\n    names = backup.namelist()\n    assert f\"{uuid}/watch.json\" in names, f\"watch.json missing from backup: {names}\"\n    assert f\"{tag_uuid}/tag.json\" in names, f\"tag.json for tag 1 missing from backup: {names}\"\n    assert f\"{tag_uuid2}/tag.json\" in names, f\"tag.json for tag 2 missing from backup: {names}\"\n\n    # --- Wipe everything ---\n    datastore.delete('all')\n    client.get(url_for(\"tags.delete_all\"), follow_redirects=True)\n\n    assert uuid not in datastore.data['watching'], \"Watch should be gone after delete\"\n    assert tag_uuid not in datastore.data['settings']['application']['tags'], \"Tag 1 should be gone after delete\"\n    assert tag_uuid2 not in datastore.data['settings']['application']['tags'], \"Tag 2 should be gone after delete\"\n\n    # --- Restore from the backup zip ---\n    res = client.post(\n        url_for(\"backups.restore.backups_restore_start\"),\n        data={\n            'zip_file': (io.BytesIO(zip_data), 'backup.zip'),\n            'include_groups': 'y',\n            'include_groups_replace_existing': 'y',\n            'include_watches': 'y',\n            'include_watches_replace_existing': 'y',\n        },\n        content_type='multipart/form-data',\n        follow_redirects=True\n    )\n    assert res.status_code == 200\n\n    # Wait for the thread to finish\n    time.sleep(2)\n\n    # --- Watch checks ---\n    restored_watch = datastore.data['watching'].get(uuid)\n    assert restored_watch is not None, f\"Watch {uuid} not found after restore\"\n    assert restored_watch['url'] == watch_url, \"Restored watch URL does not match\"\n    assert isinstance(restored_watch, Watch.model), \\\n        f\"Watch not properly rehydrated, got {type(restored_watch)}\"\n    assert restored_watch.history_n >= 1, \\\n        f\"Restored watch should have at least 1 history entry, got {restored_watch.history_n}\"\n\n    # --- Tag checks ---\n    restored_tags = datastore.data['settings']['application']['tags']\n\n    restored_tag = restored_tags.get(tag_uuid)\n    assert restored_tag is not None, f\"Tag {tag_uuid} not found after restore\"\n    assert restored_tag['title'] == \"Tasty backup tag\", \"Restored tag 1 title does not match\"\n    assert isinstance(restored_tag, Tag.model), \\\n        f\"Tag 1 not properly rehydrated, got {type(restored_tag)}\"\n\n    restored_tag2 = restored_tags.get(tag_uuid2)\n    assert restored_tag2 is not None, f\"Tag {tag_uuid2} not found after restore\"\n    assert restored_tag2['title'] == \"Tasty backup tag number two\", \"Restored tag 2 title does not match\"\n    assert isinstance(restored_tag2, Tag.model), \\\n        f\"Tag 2 not properly rehydrated, got {type(restored_tag2)}\"\n\n\ndef test_backup_restore_zip_slip_rejected(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"Zip Slip path traversal entries in a restore zip must be rejected.\"\"\"\n    import pytest\n    from changedetectionio.blueprint.backups.restore import import_from_zip\n\n    # Build a zip with a path traversal entry that would escape the extraction dir\n    malicious_zip = io.BytesIO()\n    with ZipFile(malicious_zip, 'w') as zf:\n        zf.writestr(\"../escaped.txt\", \"ATTACKER-CONTROLLED\")\n    malicious_zip.seek(0)\n\n    datastore = live_server.app.config['DATASTORE']\n\n    with pytest.raises(ValueError, match=\"Zip Slip\"):\n        import_from_zip(\n            zip_stream=malicious_zip,\n            datastore=datastore,\n            include_groups=True,\n            include_groups_replace=True,\n            include_watches=True,\n            include_watches_replace=True,\n        )\n\n\ndef test_backup_restore_zip_bomb_rejected(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"A zip whose total uncompressed size exceeds the limit must be rejected.\n\n    The guard reads file_size from the zip central-directory metadata — no\n    actual decompression happens, so this test is fast and uses minimal RAM.\n    100 KB of zeros compresses to ~100 bytes; monkeypatching the limit to\n    50 KB is enough to trigger the check without creating any large files.\n    \"\"\"\n    import pytest\n    import changedetectionio.blueprint.backups.restore as restore_mod\n    from changedetectionio.blueprint.backups.restore import import_from_zip\n\n    # ~100 KB of zeros → deflate compresses to ~100 bytes, but file_size metadata = 100 KB\n    bomb_zip = io.BytesIO()\n    with ZipFile(bomb_zip, 'w', compression=ZIP_DEFLATED) as zf:\n        zf.writestr(\"data.txt\", b\"\\x00\" * (100 * 1024))\n    bomb_zip.seek(0)\n\n    datastore = live_server.app.config['DATASTORE']\n    original_limit = restore_mod._MAX_DECOMPRESSED_BYTES\n    try:\n        restore_mod._MAX_DECOMPRESSED_BYTES = 50 * 1024  # 50 KB limit for this test\n        with pytest.raises(ValueError, match=\"decompressed size\"):\n            import_from_zip(\n                zip_stream=bomb_zip,\n                datastore=datastore,\n                include_groups=True,\n                include_groups_replace=True,\n                include_watches=True,\n                include_watches_replace=True,\n            )\n    finally:\n        restore_mod._MAX_DECOMPRESSED_BYTES = original_limit"
  },
  {
    "path": "changedetectionio/tests/test_basic_socketio.py",
    "content": "#!/usr/bin/env python3\n\nimport time\nfrom flask import url_for\nfrom .util import (\n    set_original_response,\n    set_modified_response,\n    live_server_setup,\n    wait_for_all_checks, delete_all_watches\n)\nfrom loguru import logger\n\ndef run_socketio_watch_update_test(client, live_server, password_mode=\"\", datastore_path=\"\"):\n    \"\"\"Test that the socketio emits a watch update event when content changes\"\"\"\n\n    # Set up the test server\n    set_original_response(datastore_path=datastore_path)\n\n\n    # Get the SocketIO instance from the app\n    from changedetectionio.flask_app import app\n    socketio = app.extensions['socketio']\n\n    # Create a test client for SocketIO\n    socketio_test_client = socketio.test_client(app, flask_test_client=client)\n    if password_mode == \"not logged in, should exit on connect\":\n        assert not socketio_test_client.is_connected(), \"Failed to connect to Socket.IO server because it should bounce this connect\"\n        return\n\n    assert socketio_test_client.is_connected(), \"Failed to connect to Socket.IO server\"\n    print(\"Successfully connected to Socket.IO server\")\n\n    # Add our URL to the import page\n    res = client.post(\n        url_for(\"imports.import_page\"),\n        data={\"urls\": url_for('test_endpoint', _external=True)},\n        follow_redirects=True\n    )\n    assert b\"1 Imported\" in res.data\n\n    res = client.get(url_for(\"watchlist.index\"))\n    assert url_for('test_endpoint', _external=True).encode() in res.data\n\n    # Wait for initial check to complete\n    wait_for_all_checks(client)\n\n    # Clear any initial messages\n    socketio_test_client.get_received()\n\n    # Make a change to trigger an update\n    set_modified_response(datastore_path=datastore_path)\n\n    # Force recheck\n    res = client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    assert b'Queued 1 watch for rechecking.' in res.data\n\n    # Wait for the watch to be checked\n    wait_for_all_checks(client)\n\n    has_watch_update = False\n    has_unviewed_update = False\n    got_general_stats_update = False\n\n    for i in range(10):\n        # Get received events\n        received = socketio_test_client.get_received()\n\n        if received:\n            logger.info(f\"Received {len(received)} events after {i+1} seconds\")\n            for event in received:\n                if event['name'] == 'watch_update':\n                    has_watch_update = True\n                if event['name'] == 'general_stats_update':\n                    got_general_stats_update = True\n\n        if has_unviewed_update:\n            break\n\n        # Force a recheck every 5 seconds to ensure events are emitted\n#        if i > 0 and i % 5 == 0:\n#            print(f\"Still waiting for events, forcing another recheck...\")\n#            res = client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n#            assert b'Queued 1 watch for rechecking.' in res.data\n#            wait_for_all_checks(client)\n\n#        print(f\"Waiting for unviewed update event... {i+1}/{max_wait}\")\n        time.sleep(1)\n\n    # Verify we received watch_update events\n    assert has_watch_update, \"No watch_update events received\"\n\n    # Verify we received an unviewed event\n    assert got_general_stats_update, \"Got general stats update event\"\n\n    # Alternatively, check directly if the watch in the datastore is marked as unviewed\n    from changedetectionio.flask_app import app\n    datastore = app.config.get('DATASTORE')\n\n    watch_uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))\n\n    # Get the watch from the datastore\n    watch = datastore.data['watching'].get(watch_uuid)\n    assert watch, f\"Watch {watch_uuid} not found in datastore\"\n    assert watch.has_unviewed, \"The watch was not marked as unviewed after content change\"\n\n    # Clean up\n    delete_all_watches(client)\n\ndef test_everything(live_server, client, measure_memory_usage, datastore_path):\n\n   #  live_server_setup(live_server) # Setup on conftest per function\n\n    run_socketio_watch_update_test(password_mode=\"\", live_server=live_server, client=client, datastore_path=datastore_path)\n\n    ############################ Password required auth check ##############################\n\n    # Enable password check and diff page access bypass\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\"application-password\": \"foobar\",\n              \"requests-time_between_check-minutes\": 180,\n              'application-fetch_backend': \"html_requests\"},\n        follow_redirects=True\n    )\n\n    assert b\"Password protection enabled.\" in res.data\n\n    run_socketio_watch_update_test(password_mode=\"not logged in, should exit on connect\", live_server=live_server, client=client, datastore_path=datastore_path)\n    res = client.post(\n        url_for(\"login\"),\n        data={\"password\": \"foobar\"},\n        follow_redirects=True\n    )\n\n    # Yes we are correctly logged in\n    assert b\"LOG OUT\" in res.data\n    run_socketio_watch_update_test(password_mode=\"should be like normal\", live_server=live_server, client=client, datastore_path=datastore_path)\n"
  },
  {
    "path": "changedetectionio/tests/test_block_while_text_present.py",
    "content": "#!/usr/bin/env python3\n\nimport time\nfrom flask import url_for\nfrom .util import live_server_setup, wait_for_all_checks, delete_all_watches\nfrom changedetectionio import html_tools\nimport os\n\ndef set_original_ignore_response(datastore_path):\n\n    test_return_data = \"\"\"\n       \n     Some initial text
    \n     
    Which is across multiple lines
    \n     
    \n     So let's see what happens.  
    \n     \n     \n\n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n\n\ndef set_modified_original_ignore_response(datastore_path):\n    test_return_data = \"\"\"\n       \n     Some NEW nice initial text
    \n     
    Which is across multiple lines
    \n     
    \n     So let's see what happens.  
    \n     
    new ignore stuff
    \n     
    out of stock
    \n     
    blah
    \n     \n     \n\n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n\n\n# Is the same but includes ZZZZZ, 'ZZZZZ' is the last line in ignore_text\ndef set_modified_response_minus_block_text(datastore_path):\n    test_return_data = \"\"\"\n       \n     Some NEW nice initial text
    \n     
    Which is across multiple lines
    \n     
    now on sale $2/p>\n     
    \n     So let's see what happens.  
    \n     
    new ignore stuff
    \n     
    blah
    \n     \n     \n\n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n\n\ndef test_check_block_changedetection_text_NOT_present(client, live_server, measure_memory_usage, datastore_path):\n\n    # Use a mix of case in ZzZ to prove it works case-insensitive.\n    ignore_text = \"out of stoCk\\r\\nfoobar\"\n    set_original_ignore_response(datastore_path=datastore_path)\n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))\n    # Give the thread time to pick it up\n    wait_for_all_checks(client)\n\n    # Goto the edit page, add our ignore text\n    # Add our URL to the import page\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=uuid),\n        data={\"text_should_not_be_present\": ignore_text,\n              \"url\": test_url,\n              'fetch_backend': \"html_requests\",\n              \"time_between_check_use_default\": \"y\"\n              },\n        follow_redirects=True\n    )\n    assert b\"Updated watch.\" in res.data\n\n    # Give the thread time to pick it up\n    wait_for_all_checks(client)\n    # Check it saved\n    res = client.get(\n        url_for(\"ui.ui_edit.edit_page\", uuid=uuid),\n    )\n    assert bytes(ignore_text.encode('utf-8')) in res.data\n\n    # Trigger a check\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    # Give the thread time to pick it up\n    wait_for_all_checks(client)\n\n    # It should report nothing found (no new 'has-unread-changes' class)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'has-unread-changes' not in res.data\n    assert b'/test-endpoint' in res.data\n\n    # The page changed, BUT the text is still there, just the rest of it changes, we should not see a change\n    set_modified_original_ignore_response(datastore_path=datastore_path)\n\n    # Trigger a check\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    # Give the thread time to pick it up\n    wait_for_all_checks(client)\n\n    # It should report nothing found (no new 'has-unread-changes' class)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'has-unread-changes' not in res.data\n    assert b'/test-endpoint' in res.data\n\n    # 2548\n    # Going back to the ORIGINAL should NOT trigger a change\n    set_original_ignore_response(datastore_path=datastore_path)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'has-unread-changes' not in res.data\n\n    # Now we set a change where the text is gone AND its different content, it should now trigger\n    set_modified_response_minus_block_text(datastore_path=datastore_path)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"watchlist.index\"))\n\n    assert b'has-unread-changes' in res.data\n\n    # Clearing all history then viewing it should show us what is blocked\n    set_modified_original_ignore_response(datastore_path=datastore_path)\n    client.get(url_for(\"ui.clear_watch_history\", uuid=uuid))\n    wait_for_all_checks(client)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n    res = client.get(\n        url_for(\"ui.ui_preview.preview_page\", uuid=uuid)\n    )\n    assert b'blocked_line_numbers = [10]' in res.data\n\n\n    delete_all_watches(client)\n"
  },
  {
    "path": "changedetectionio/tests/test_clone.py",
    "content": "#!/usr/bin/env python3\n\nimport time\nfrom flask import url_for\nfrom .util import live_server_setup, wait_for_all_checks\nimport os\n\n\ndef test_clone_functionality(client, live_server, measure_memory_usage, datastore_path):\n\n   #  live_server_setup(live_server) # Setup on conftest per function\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(\"Some content\")\n\n    test_url = url_for('test_endpoint', _external=True)\n\n    # Add our URL to the import page\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    # So that we can be sure the same history doesnt carry over\n    time.sleep(1)\n\n    res = client.get(\n        url_for(\"ui.form_clone\", uuid=\"first\"),\n        follow_redirects=True\n    )\n    existing_uuids = set()\n\n    for uuid, watch in live_server.app.config['DATASTORE'].data['watching'].items():\n        new_uuids = set(watch.history.keys())\n        duplicates = existing_uuids.intersection(new_uuids)\n        assert len(duplicates) == 0\n        existing_uuids.update(new_uuids)\n\n    assert b\"Cloned\" in res.data\n"
  },
  {
    "path": "changedetectionio/tests/test_commit_persistence.py",
    "content": "#!/usr/bin/env python3\n\"\"\"\nTests for immediate commit-based persistence system.\n\nTests cover:\n- Watch.commit() persistence to disk\n- Concurrent commit safety (race conditions)\n- Processor config separation\n- Data loss prevention (settings, tags, watch modifications)\n\"\"\"\n\nimport json\nimport os\nimport threading\nimport time\nfrom flask import url_for\nfrom .util import wait_for_all_checks\n\n\n# ==============================================================================\n# 2. Commit() Persistence Tests\n# ==============================================================================\n\ndef test_watch_commit_persists_to_disk(client, live_server):\n    \"\"\"Test that watch.commit() actually writes to watch.json immediately\"\"\"\n    datastore = client.application.config.get('DATASTORE')\n\n    # Create a watch\n    uuid = datastore.add_watch(url='http://example.com', extras={'title': 'Original Title'})\n    watch = datastore.data['watching'][uuid]\n\n    # Modify and commit\n    watch['title'] = 'Modified Title'\n    watch['paused'] = True\n    watch.commit()\n\n    # Read directly from disk (bypass datastore cache)\n    watch_json_path = os.path.join(watch.data_dir, 'watch.json')\n    assert os.path.exists(watch_json_path), \"watch.json should exist on disk\"\n\n    with open(watch_json_path, 'r') as f:\n        disk_data = json.load(f)\n\n    assert disk_data['title'] == 'Modified Title', \"Title should be persisted to disk\"\n    assert disk_data['paused'] == True, \"Paused state should be persisted to disk\"\n    assert disk_data['uuid'] == uuid, \"UUID should match\"\n\n\ndef test_watch_commit_survives_reload(client, live_server):\n    \"\"\"Test that committed changes survive datastore reload\"\"\"\n    from changedetectionio.store import ChangeDetectionStore\n\n    datastore = client.application.config.get('DATASTORE')\n    datastore_path = datastore.datastore_path\n\n    # Create and modify a watch\n    uuid = datastore.add_watch(url='http://example.com', extras={'title': 'Test Watch'})\n    watch = datastore.data['watching'][uuid]\n    watch['title'] = 'Persisted Title'\n    watch['paused'] = True\n    watch['tags'] = ['tag-1', 'tag-2']\n    watch.commit()\n\n    # Simulate app restart - create new datastore instance\n    datastore2 = ChangeDetectionStore(datastore_path=datastore_path)\n    datastore2.reload_state(\n        datastore_path=datastore_path,\n        include_default_watches=False,\n        version_tag='test'\n    )\n\n    # Check data survived\n    assert uuid in datastore2.data['watching'], \"Watch should exist after reload\"\n    reloaded_watch = datastore2.data['watching'][uuid]\n    assert reloaded_watch['title'] == 'Persisted Title', \"Title should survive reload\"\n    assert reloaded_watch['paused'] == True, \"Paused state should survive reload\"\n    assert reloaded_watch['tags'] == ['tag-1', 'tag-2'], \"Tags should survive reload\"\n\n\ndef test_watch_commit_atomic_on_crash(client, live_server):\n    \"\"\"Test that atomic writes prevent corruption (temp file pattern)\"\"\"\n    datastore = client.application.config.get('DATASTORE')\n\n    uuid = datastore.add_watch(url='http://example.com', extras={'title': 'Original'})\n    watch = datastore.data['watching'][uuid]\n\n    # First successful commit\n    watch['title'] = 'First Save'\n    watch.commit()\n\n    # Verify watch.json exists and is valid\n    watch_json_path = os.path.join(watch.data_dir, 'watch.json')\n    with open(watch_json_path, 'r') as f:\n        data = json.load(f)  # Should not raise JSONDecodeError\n        assert data['title'] == 'First Save'\n\n    # Second commit - even if interrupted, original file should be intact\n    # (atomic write uses temp file + rename, so original is never corrupted)\n    watch['title'] = 'Second Save'\n    watch.commit()\n\n    with open(watch_json_path, 'r') as f:\n        data = json.load(f)\n        assert data['title'] == 'Second Save'\n\n\ndef test_multiple_watches_commit_independently(client, live_server):\n    \"\"\"Test that committing one watch doesn't affect others\"\"\"\n    datastore = client.application.config.get('DATASTORE')\n\n    # Create multiple watches\n    uuid1 = datastore.add_watch(url='http://example1.com', extras={'title': 'Watch 1'})\n    uuid2 = datastore.add_watch(url='http://example2.com', extras={'title': 'Watch 2'})\n    uuid3 = datastore.add_watch(url='http://example3.com', extras={'title': 'Watch 3'})\n\n    watch1 = datastore.data['watching'][uuid1]\n    watch2 = datastore.data['watching'][uuid2]\n    watch3 = datastore.data['watching'][uuid3]\n\n    # Modify and commit only watch2\n    watch2['title'] = 'Modified Watch 2'\n    watch2['paused'] = True\n    watch2.commit()\n\n    # Read all from disk\n    def read_watch_json(uuid):\n        watch = datastore.data['watching'][uuid]\n        path = os.path.join(watch.data_dir, 'watch.json')\n        with open(path, 'r') as f:\n            return json.load(f)\n\n    data1 = read_watch_json(uuid1)\n    data2 = read_watch_json(uuid2)\n    data3 = read_watch_json(uuid3)\n\n    # Only watch2 should have changes\n    assert data1['title'] == 'Watch 1', \"Watch 1 should be unchanged\"\n    assert data1['paused'] == False, \"Watch 1 should not be paused\"\n\n    assert data2['title'] == 'Modified Watch 2', \"Watch 2 should be modified\"\n    assert data2['paused'] == True, \"Watch 2 should be paused\"\n\n    assert data3['title'] == 'Watch 3', \"Watch 3 should be unchanged\"\n    assert data3['paused'] == False, \"Watch 3 should not be paused\"\n\n\n# ==============================================================================\n# 3. Concurrency/Race Condition Tests\n# ==============================================================================\n\ndef test_concurrent_watch_commits_dont_corrupt(client, live_server):\n    \"\"\"Test that simultaneous commits to same watch don't corrupt JSON\"\"\"\n    datastore = client.application.config.get('DATASTORE')\n\n    uuid = datastore.add_watch(url='http://example.com', extras={'title': 'Test'})\n    watch = datastore.data['watching'][uuid]\n\n    errors = []\n\n    def modify_and_commit(field, value):\n        try:\n            watch[field] = value\n            watch.commit()\n        except Exception as e:\n            errors.append(e)\n\n    # Run 10 concurrent commits\n    threads = []\n    for i in range(10):\n        t = threading.Thread(target=modify_and_commit, args=('title', f'Title {i}'))\n        threads.append(t)\n        t.start()\n\n    for t in threads:\n        t.join()\n\n    # Should not have any errors\n    assert len(errors) == 0, f\"Expected no errors, got: {errors}\"\n\n    # JSON file should still be valid (not corrupted)\n    watch_json_path = os.path.join(watch.data_dir, 'watch.json')\n    with open(watch_json_path, 'r') as f:\n        data = json.load(f)  # Should not raise JSONDecodeError\n        assert data['uuid'] == uuid, \"UUID should still be correct\"\n        assert 'Title' in data['title'], \"Title should contain 'Title'\"\n\n\ndef test_concurrent_modifications_during_commit(client, live_server):\n    \"\"\"Test that modifying watch during commit doesn't cause RuntimeError\"\"\"\n    datastore = client.application.config.get('DATASTORE')\n\n    uuid = datastore.add_watch(url='http://example.com', extras={'title': 'Test'})\n    watch = datastore.data['watching'][uuid]\n\n    errors = []\n    stop_flag = threading.Event()\n\n    def keep_modifying():\n        \"\"\"Continuously modify watch\"\"\"\n        try:\n            i = 0\n            while not stop_flag.is_set():\n                watch['title'] = f'Title {i}'\n                watch['paused'] = i % 2 == 0\n                i += 1\n                time.sleep(0.001)\n        except Exception as e:\n            errors.append(('modifier', e))\n\n    def keep_committing():\n        \"\"\"Continuously commit watch\"\"\"\n        try:\n            for _ in range(20):\n                watch.commit()\n                time.sleep(0.005)\n        except Exception as e:\n            errors.append(('committer', e))\n\n    # Start concurrent modification and commits\n    modifier = threading.Thread(target=keep_modifying)\n    committer = threading.Thread(target=keep_committing)\n\n    modifier.start()\n    committer.start()\n\n    committer.join()\n    stop_flag.set()\n    modifier.join()\n\n    # Should not have RuntimeError from dict changing during iteration\n    runtime_errors = [e for source, e in errors if isinstance(e, RuntimeError)]\n    assert len(runtime_errors) == 0, f\"Should not have RuntimeError, got: {runtime_errors}\"\n\n\ndef test_datastore_lock_protects_commit_snapshot(client, live_server):\n    \"\"\"Test that datastore.lock prevents race conditions during deepcopy\"\"\"\n    datastore = client.application.config.get('DATASTORE')\n\n    uuid = datastore.add_watch(url='http://example.com', extras={'title': 'Test'})\n    watch = datastore.data['watching'][uuid]\n\n    # Add some complex nested data\n    watch['browser_steps'] = [\n        {'operation': 'click', 'selector': '#foo'},\n        {'operation': 'wait', 'seconds': 5}\n    ]\n\n    errors = []\n    commits_succeeded = [0]\n\n    def rapid_commits():\n        try:\n            for i in range(50):\n                watch['title'] = f'Title {i}'\n                watch.commit()\n                commits_succeeded[0] += 1\n                time.sleep(0.001)\n        except Exception as e:\n            errors.append(e)\n\n    # Multiple threads doing rapid commits\n    threads = [threading.Thread(target=rapid_commits) for _ in range(3)]\n\n    for t in threads:\n        t.start()\n    for t in threads:\n        t.join()\n\n    assert len(errors) == 0, f\"Expected no errors, got: {errors}\"\n    assert commits_succeeded[0] == 150, f\"Expected 150 commits, got {commits_succeeded[0]}\"\n\n    # Final JSON should be valid\n    watch_json_path = os.path.join(watch.data_dir, 'watch.json')\n    with open(watch_json_path, 'r') as f:\n        data = json.load(f)\n        assert data['uuid'] == uuid\n\n\n# ==============================================================================\n# 4. Processor Config Separation Tests\n# ==============================================================================\n\ndef test_processor_config_never_in_watch_json(client, live_server):\n    \"\"\"Test that processor_config_* fields are filtered out of watch.json\"\"\"\n    datastore = client.application.config.get('DATASTORE')\n\n    uuid = datastore.add_watch(\n        url='http://example.com',\n        extras={\n            'title': 'Test Watch',\n            'processor': 'restock_diff'\n        }\n    )\n\n    watch = datastore.data['watching'][uuid]\n\n    # Try to set processor config fields (these should be filtered during commit)\n    watch['processor_config_price_threshold'] = 10.0\n    watch['processor_config_some_setting'] = 'value'\n    watch['processor_config_another'] = {'nested': 'data'}\n    watch.commit()\n\n    # Read watch.json from disk\n    watch_json_path = os.path.join(watch.data_dir, 'watch.json')\n    with open(watch_json_path, 'r') as f:\n        data = json.load(f)\n\n    # Verify processor_config_* fields are NOT in watch.json\n    for key in data.keys():\n        assert not key.startswith('processor_config_'), \\\n            f\"Found {key} in watch.json - processor configs should be in separate file!\"\n\n    # Normal fields should still be there\n    assert data['title'] == 'Test Watch'\n    assert data['processor'] == 'restock_diff'\n\n\ndef test_api_post_saves_processor_config_separately(client, live_server):\n    \"\"\"Test that API POST saves processor configs to {processor}.json\"\"\"\n    import json\n    from changedetectionio.processors import extract_processor_config_from_form_data\n\n    # Get API key\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n\n    # Create watch via API with processor config\n    response = client.post(\n        url_for(\"createwatch\"),\n        data=json.dumps({\n            'url': 'http://example.com',\n            'processor': 'restock_diff',\n            'processor_config_price_threshold': 10.0,\n            'processor_config_in_stock_only': True\n        }),\n        headers={'content-type': 'application/json', 'x-api-key': api_key}\n    )\n\n    assert response.status_code in (200, 201), f\"Expected 200/201, got {response.status_code}\"\n    uuid = response.json.get('uuid')\n    assert uuid, \"Should return UUID\"\n\n    datastore = client.application.config.get('DATASTORE')\n    watch = datastore.data['watching'][uuid]\n\n    # Check that processor config file exists\n    processor_config_path = os.path.join(watch.data_dir, 'restock_diff.json')\n    assert os.path.exists(processor_config_path), \"Processor config file should exist\"\n\n    with open(processor_config_path, 'r') as f:\n        config = json.load(f)\n\n    # Verify fields are saved WITHOUT processor_config_ prefix\n    assert config.get('price_threshold') == 10.0, \"Should have price_threshold (no prefix)\"\n    assert config.get('in_stock_only') == True, \"Should have in_stock_only (no prefix)\"\n    assert 'processor_config_price_threshold' not in config, \"Should NOT have prefixed keys\"\n\n\ndef test_api_put_saves_processor_config_separately(client, live_server):\n    \"\"\"Test that API PUT updates processor configs in {processor}.json\"\"\"\n    import json\n    datastore = client.application.config.get('DATASTORE')\n\n    # Get API key\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n\n    # Create watch\n    uuid = datastore.add_watch(\n        url='http://example.com',\n        extras={'processor': 'restock_diff'}\n    )\n\n    # Update via API with processor config\n    response = client.put(\n        url_for(\"watch\", uuid=uuid),\n        data=json.dumps({\n            'processor_config_price_threshold': 15.0,\n            'processor_config_min_stock': 5\n        }),\n        headers={'content-type': 'application/json', 'x-api-key': api_key}\n    )\n\n    # PUT might return different status codes, 200 or 204 are both OK\n    assert response.status_code in (200, 204), f\"Expected 200/204, got {response.status_code}: {response.data}\"\n\n    watch = datastore.data['watching'][uuid]\n\n    # Check processor config file\n    processor_config_path = os.path.join(watch.data_dir, 'restock_diff.json')\n    assert os.path.exists(processor_config_path), \"Processor config file should exist\"\n\n    with open(processor_config_path, 'r') as f:\n        config = json.load(f)\n\n    assert config.get('price_threshold') == 15.0, \"Should have updated price_threshold\"\n    assert config.get('min_stock') == 5, \"Should have min_stock\"\n\n\ndef test_ui_edit_saves_processor_config_separately(client, live_server):\n    \"\"\"Test that processor_config_* fields never appear in watch.json (even from UI)\"\"\"\n    datastore = client.application.config.get('DATASTORE')\n\n    # Create watch\n    uuid = datastore.add_watch(\n        url='http://example.com',\n        extras={'processor': 'text_json_diff', 'title': 'Test'}\n    )\n\n    watch = datastore.data['watching'][uuid]\n\n    # Simulate someone accidentally trying to set processor_config fields directly\n    watch['processor_config_should_not_save'] = 'test_value'\n    watch['processor_config_another_field'] = 123\n    watch['normal_field'] = 'this_should_save'\n    watch.commit()\n\n    # Check watch.json has NO processor_config_* fields (main point of this test)\n    watch_json_path = os.path.join(watch.data_dir, 'watch.json')\n    with open(watch_json_path, 'r') as f:\n        watch_data = json.load(f)\n\n    for key in watch_data.keys():\n        assert not key.startswith('processor_config_'), \\\n            f\"Found {key} in watch.json - processor configs should be filtered during commit\"\n\n    # Verify normal fields still save\n    assert watch_data['normal_field'] == 'this_should_save', \"Normal fields should save\"\n    assert watch_data['title'] == 'Test', \"Original fields should still be there\"\n\n\ndef test_browser_steps_normalized_to_empty_list(client, live_server):\n    \"\"\"Test that meaningless browser_steps are normalized to [] during commit\"\"\"\n    datastore = client.application.config.get('DATASTORE')\n\n    uuid = datastore.add_watch(url='http://example.com')\n    watch = datastore.data['watching'][uuid]\n\n    # Set browser_steps to meaningless values\n    watch['browser_steps'] = [\n        {'operation': 'Choose one', 'selector': ''},\n        {'operation': 'Goto site', 'selector': ''},\n        {'operation': '', 'selector': '#foo'}\n    ]\n    watch.commit()\n\n    # Read from disk\n    watch_json_path = os.path.join(watch.data_dir, 'watch.json')\n    with open(watch_json_path, 'r') as f:\n        data = json.load(f)\n\n    # Should be normalized to empty list\n    assert data['browser_steps'] == [], \"Meaningless browser_steps should be normalized to []\"\n\n\n# ==============================================================================\n# 5. Data Loss Prevention Tests\n# ==============================================================================\n\ndef test_settings_persist_after_update(client, live_server):\n    \"\"\"Test that settings updates are committed and survive restart\"\"\"\n    from changedetectionio.store import ChangeDetectionStore\n\n    datastore = client.application.config.get('DATASTORE')\n    datastore_path = datastore.datastore_path\n\n    # Update settings directly (bypass form validation issues)\n    datastore.data['settings']['application']['empty_pages_are_a_change'] = True\n    datastore.data['settings']['application']['fetch_backend'] = 'html_requests'\n    datastore.data['settings']['requests']['time_between_check']['minutes'] = 120\n    datastore.commit()\n\n    # Simulate restart\n    datastore2 = ChangeDetectionStore(datastore_path=datastore_path)\n    datastore2.reload_state(\n        datastore_path=datastore_path,\n        include_default_watches=False,\n        version_tag='test'\n    )\n\n    # Verify settings survived\n    assert datastore2.data['settings']['application']['empty_pages_are_a_change'] == True, \"empty_pages_are_a_change should persist\"\n    assert datastore2.data['settings']['application']['fetch_backend'] == 'html_requests', \"fetch_backend should persist\"\n    assert datastore2.data['settings']['requests']['time_between_check']['minutes'] == 120, \"time_between_check should persist\"\n\n\ndef test_tag_mute_persists(client, live_server):\n    \"\"\"Test that tag mute/unmute operations persist\"\"\"\n    from changedetectionio.store import ChangeDetectionStore\n\n    datastore = client.application.config.get('DATASTORE')\n    datastore_path = datastore.datastore_path\n\n    # Add a tag\n    tag_uuid = datastore.add_tag('Test Tag')\n\n    # Mute the tag\n    response = client.get(url_for(\"tags.mute\", uuid=tag_uuid))\n    assert response.status_code == 302  # Redirect\n\n    # Verify muted in memory\n    assert datastore.data['settings']['application']['tags'][tag_uuid]['notification_muted'] == True\n\n    # Simulate restart\n    datastore2 = ChangeDetectionStore(datastore_path=datastore_path)\n    datastore2.reload_state(\n        datastore_path=datastore_path,\n        include_default_watches=False,\n        version_tag='test'\n    )\n\n    # Verify mute state survived\n    assert tag_uuid in datastore2.data['settings']['application']['tags']\n    assert datastore2.data['settings']['application']['tags'][tag_uuid]['notification_muted'] == True\n\n\ndef test_tag_delete_removes_from_watches(client, live_server):\n    \"\"\"Test that deleting a tag removes it from all watches\"\"\"\n    datastore = client.application.config.get('DATASTORE')\n\n    # Create a tag\n    tag_uuid = datastore.add_tag('Test Tag')\n\n    # Create watches with this tag\n    uuid1 = datastore.add_watch(url='http://example1.com')\n    uuid2 = datastore.add_watch(url='http://example2.com')\n    uuid3 = datastore.add_watch(url='http://example3.com')\n\n    watch1 = datastore.data['watching'][uuid1]\n    watch2 = datastore.data['watching'][uuid2]\n    watch3 = datastore.data['watching'][uuid3]\n\n    watch1['tags'] = [tag_uuid]\n    watch1.commit()\n    watch2['tags'] = [tag_uuid, 'other-tag']\n    watch2.commit()\n    # watch3 has no tags\n\n    # Delete the tag\n    response = client.get(url_for(\"tags.delete\", uuid=tag_uuid))\n    assert response.status_code == 302\n\n    # Wait for background thread to complete\n    time.sleep(1)\n\n    # Tag should be removed from settings\n    assert tag_uuid not in datastore.data['settings']['application']['tags']\n\n    # Tag should be removed from watches and persisted\n    def check_watch_tags(uuid):\n        watch = datastore.data['watching'][uuid]\n        watch_json_path = os.path.join(watch.data_dir, 'watch.json')\n        with open(watch_json_path, 'r') as f:\n            return json.load(f)['tags']\n\n    assert tag_uuid not in check_watch_tags(uuid1), \"Tag should be removed from watch1\"\n    assert tag_uuid not in check_watch_tags(uuid2), \"Tag should be removed from watch2\"\n    assert 'other-tag' in check_watch_tags(uuid2), \"Other tags should remain in watch2\"\n    assert check_watch_tags(uuid3) == [], \"Watch3 should still have empty tags\"\n\n\ndef test_watch_pause_unpause_persists(client, live_server):\n    \"\"\"Test that pause/unpause operations commit and persist\"\"\"\n    datastore = client.application.config.get('DATASTORE')\n\n    # Get API key\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n\n    uuid = datastore.add_watch(url='http://example.com')\n    watch = datastore.data['watching'][uuid]\n\n    # Pause via API\n    response = client.get(url_for(\"watch\", uuid=uuid, paused='paused'), headers={'x-api-key': api_key})\n    assert response.status_code == 200\n\n    # Check persisted to disk\n    watch_json_path = os.path.join(watch.data_dir, 'watch.json')\n    with open(watch_json_path, 'r') as f:\n        data = json.load(f)\n    assert data['paused'] == True, \"Pause should be persisted\"\n\n    # Unpause\n    response = client.get(url_for(\"watch\", uuid=uuid, paused='unpaused'), headers={'x-api-key': api_key})\n    assert response.status_code == 200\n\n    with open(watch_json_path, 'r') as f:\n        data = json.load(f)\n    assert data['paused'] == False, \"Unpause should be persisted\"\n\n\ndef test_watch_mute_unmute_persists(client, live_server):\n    \"\"\"Test that mute/unmute operations commit and persist\"\"\"\n    datastore = client.application.config.get('DATASTORE')\n\n    # Get API key\n    api_key = live_server.app.config['DATASTORE'].data['settings']['application'].get('api_access_token')\n\n    uuid = datastore.add_watch(url='http://example.com')\n    watch = datastore.data['watching'][uuid]\n\n    # Mute via API\n    response = client.get(url_for(\"watch\", uuid=uuid, muted='muted'), headers={'x-api-key': api_key})\n    assert response.status_code == 200\n\n    # Check persisted to disk\n    watch_json_path = os.path.join(watch.data_dir, 'watch.json')\n    with open(watch_json_path, 'r') as f:\n        data = json.load(f)\n    assert data['notification_muted'] == True, \"Mute should be persisted\"\n\n    # Unmute\n    response = client.get(url_for(\"watch\", uuid=uuid, muted='unmuted'), headers={'x-api-key': api_key})\n    assert response.status_code == 200\n\n    with open(watch_json_path, 'r') as f:\n        data = json.load(f)\n    assert data['notification_muted'] == False, \"Unmute should be persisted\"\n\n\ndef test_ui_watch_edit_persists_all_fields(client, live_server):\n    \"\"\"Test that UI watch edit form persists all modified fields\"\"\"\n    from changedetectionio.store import ChangeDetectionStore\n\n    datastore = client.application.config.get('DATASTORE')\n    datastore_path = datastore.datastore_path\n\n    # Create watch\n    uuid = datastore.add_watch(url='http://example.com')\n\n    # Edit via UI with multiple field changes\n    response = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=uuid),\n        data={\n            'url': 'http://updated-example.com',\n            'title': 'Updated Watch Title',\n            'time_between_check-hours': '2',\n            'time_between_check-minutes': '30',\n            'include_filters': '#content',\n            'fetch_backend': 'html_requests',\n            'method': 'POST',\n            'ignore_text': 'Advertisement\\nTracking'\n        },\n        follow_redirects=True\n    )\n\n    assert b\"Updated watch\" in response.data or b\"Saved\" in response.data\n\n    # Simulate restart\n    datastore2 = ChangeDetectionStore(datastore_path=datastore_path)\n    datastore2.reload_state(\n        datastore_path=datastore_path,\n        include_default_watches=False,\n        version_tag='test'\n    )\n\n    # Verify all fields survived\n    watch = datastore2.data['watching'][uuid]\n    assert watch['url'] == 'http://updated-example.com'\n    assert watch['title'] == 'Updated Watch Title'\n    assert watch['time_between_check']['hours'] == 2\n    assert watch['time_between_check']['minutes'] == 30\n    assert watch['fetch_backend'] == 'html_requests'\n    assert watch['method'] == 'POST'\n"
  },
  {
    "path": "changedetectionio/tests/test_conditions.py",
    "content": "#!/usr/bin/env python3\nimport json\nimport time\nimport os\n\nfrom flask import url_for\nfrom .util import live_server_setup, wait_for_all_checks, delete_all_watches\nfrom ..model import CONDITIONS_MATCH_LOGIC_DEFAULT\n\n\ndef set_original_response(datastore_path, number=\"50\"):\n    test_return_data = f\"\"\"\n       \n     
    Test Page for Conditions
    \n     
    This page contains a number that will be tested with conditions.
    \n     
    Current value: {number}
    \n     \n     \n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n\ndef set_number_in_range_response(datastore_path, number=\"75\"):\n    test_return_data = f\"\"\"\n       \n     
    Test Page for Conditions
    \n     
    This page contains a number that will be tested with conditions.
    \n     
    Current value: {number}
    \n     \n     \n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n\ndef set_number_out_of_range_response(datastore_path, number=\"150\"):\n    test_return_data = f\"\"\"\n       \n     
    Test Page for Conditions
    \n     
    This page contains a number that will be tested with conditions.
    \n     
    Current value: {number}
    \n     \n     \n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n\n\n# def test_setup(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"Test that both text and number conditions work together with AND logic.\"\"\"\n   #  live_server_setup(live_server) # Setup on conftest per function\n\ndef test_conditions_with_text_and_number(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"Test that both text and number conditions work together with AND logic.\"\"\"\n    \n    set_original_response(datastore_path=datastore_path, number=\"50\")\n    \n\n    test_url = url_for('test_endpoint', _external=True)\n\n    # Add our URL to the import page\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    # Configure the watch with two conditions connected with AND:\n    # 1. The page filtered text must contain \"5\" (first digit of value)\n    # 2. The extracted number should be >= 20 and <= 100\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=uuid),\n        data={\n            \"url\": test_url,\n            \"fetch_backend\": \"html_requests\",\n            \"include_filters\": \".number-container\",\n            \"title\": \"Number AND Text Condition Test\",\n            \"conditions_match_logic\": CONDITIONS_MATCH_LOGIC_DEFAULT,  # ALL = AND logic\n            \"conditions-0-operator\": \"in\",\n            \"conditions-0-field\": \"page_filtered_text\",\n            \"conditions-0-value\": \"5\",\n\n            \"conditions-1-operator\": \">=\",\n            \"conditions-1-field\": \"extracted_number\",\n            \"conditions-1-value\": \"20\",\n\n            \"conditions-2-operator\": \"<=\",\n            \"conditions-2-field\": \"extracted_number\",\n            \"conditions-2-value\": \"100\",\n\n            # So that 'operations' from pluggy discovery are tested\n            \"conditions-3-operator\": \"length_min\",\n            \"conditions-3-field\": \"page_filtered_text\",\n            \"conditions-3-value\": \"1\",\n\n            # So that 'operations' from pluggy discovery are tested\n            \"conditions-4-operator\": \"length_max\",\n            \"conditions-4-field\": \"page_filtered_text\",\n            \"conditions-4-value\": \"100\",\n\n            # So that 'operations' from pluggy discovery are tested\n            \"conditions-5-operator\": \"contains_regex\",\n            \"conditions-5-field\": \"page_filtered_text\",\n            \"conditions-5-value\": \"\\d\",\n            \"time_between_check_use_default\": \"y\",\n        },\n        follow_redirects=True\n    )\n    assert b\"Updated watch.\" in res.data\n\n    wait_for_all_checks(client)\n    client.get(url_for(\"ui.mark_all_viewed\"), follow_redirects=True)\n    time.sleep(1)\n\n    # Case 1\n    set_number_in_range_response(datastore_path=datastore_path, number=\"70.5\")\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    # 75 is > 20 and < 100 and contains \"5\"\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'has-unread-changes' in res.data\n\n    # Case 2: Change with one condition violated\n    # Number out of range (150) but contains '5'\n    client.get(url_for(\"ui.mark_all_viewed\"), follow_redirects=True)\n\n    set_number_out_of_range_response(datastore_path=datastore_path, number=\"150.5\")\n\n\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    # Should NOT be marked as having changes since not all conditions are met\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'has-unread-changes' not in res.data\n\n    delete_all_watches(client)\n\n# The 'validate' button next to each rule row\ndef test_condition_validate_rule_row(client, live_server, measure_memory_usage, datastore_path):\n\n    set_original_response(datastore_path=datastore_path, number=\"50\")\n\n    test_url = url_for('test_endpoint', _external=True)\n\n    # Add our URL to the import page\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n\n    # the front end submits the current form state which should override the watch in a temporary copy\n    res = client.post(\n        url_for(\"conditions.verify_condition_single_rule\", watch_uuid=uuid),  # Base URL\n        query_string={\"rule\": json.dumps({\"field\": \"extracted_number\", \"operator\": \"==\", \"value\": \"50\"})},\n        data={'include_filter': \"\"},\n        follow_redirects=True\n    )\n    assert res.status_code == 200\n    assert b'success' in res.data\n\n    # Now a number that does not equal what is found in the last fetch\n    res = client.post(\n        url_for(\"conditions.verify_condition_single_rule\", watch_uuid=uuid),  # Base URL\n        query_string={\"rule\": json.dumps({\"field\": \"extracted_number\", \"operator\": \"==\", \"value\": \"111111\"})},\n        data={'include_filter': \"\"},\n        follow_redirects=True\n    )\n    assert res.status_code == 200\n    assert b'false' in res.data\n\n    # Now custom filter that exists\n    res = client.post(\n        url_for(\"conditions.verify_condition_single_rule\", watch_uuid=uuid),  # Base URL\n        query_string={\"rule\": json.dumps({\"field\": \"extracted_number\", \"operator\": \"==\", \"value\": \"50\"})},\n        data={'include_filter': \".number-container\"},\n        follow_redirects=True\n    )\n    assert res.status_code == 200\n    assert b'success' in res.data\n\n    # Now custom filter that DOES NOT exists\n    res = client.post(\n        url_for(\"conditions.verify_condition_single_rule\", watch_uuid=uuid),  # Base URL\n        query_string={\"rule\": json.dumps({\"field\": \"extracted_number\", \"operator\": \"==\", \"value\": \"50\"})},\n        data={'include_filters': \".NOT-container\"},\n        follow_redirects=True\n    )\n    assert res.status_code == 200\n    assert b'false' in res.data\n\n    delete_all_watches(client)\n\n\n# If there was only a change in the whitespacing, then we shouldnt have a change detected\ndef test_wordcount_conditions_plugin(client, live_server, measure_memory_usage, datastore_path):\n    \n\n    test_return_data = \"\"\"\n       \n     Some initial text
    \n     
    Which is across multiple lines
    \n     
    \n     So let's see what happens.  
    \n     \n     \n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    # Give the thread time to pick it up\n    wait_for_all_checks(client)\n\n    # Check it saved\n    res = client.get(\n        url_for(\"ui.ui_edit.edit_page\", uuid=uuid),\n    )\n\n    # Assert the word count is counted correctly\n    assert b'    		13     
    \n  \n    \n    \n    \n  \n  \n    \n    \n    \n  \n  \n    \n    \n    \n  \n
    Person 1Person 2Person 3
    EmilTobiasLinus
    161410
    \n\n\n\"\"\"\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(data)\n\n\ndef set_original_response(datastore_path):\n    test_return_data = \"\"\"\n    
    \n    
    Header
    \n    
    \n    \n       \n     Some initial text
    \n     
    Which is across multiple lines
    \n     
    \n     So let's see what happens.  
    \n    
    Some text that will change
    \n     \n    
    \n    
    Footer
    \n    
    \n     \n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n\n\ndef set_modified_response(datastore_path):\n    test_return_data = \"\"\"\n    
    \n    
    Header changed
    \n    
    \n    \n       \n     Some initial text
    \n     
    Which is across multiple lines
    \n     
    \n     So let's see what happens.  
    \n    
    Some text that changes
    \n     \n    
    \n    
    Footer changed
    \n    
    \n     \n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n\n\ndef test_element_removal_output():\n    from inscriptis import get_text\n\n    # Check text with sub-parts renders correctly\n    content = \"\"\"\n    
    \n    
    Header
    \n    
    \n    \n       \n     Some initial text
    \n     
    across multiple lines
    \n     
    Some text that changes
    \n     
    Some text should be matched by xPath // selector
    \n     
    Some text should be matched by xPath selector
    \n     
    Some text should be matched by xPath1 selector
    \n     \n    
    \n    
    Footer
    \n    
    \n     \n    \"\"\"\n    html_blob = element_removal(\n      [\n        \"header\",\n        \"footer\",\n        \"nav\",\n        \"#changetext\",\n        \"//*[contains(text(), 'xPath // selector')]\",\n        \"xpath://*[contains(text(), 'xPath selector')]\",\n        \"xpath1://*[contains(text(), 'xPath1 selector')]\"\n      ],\n      html_content=content\n    )\n    text = get_text(html_blob)\n    assert (\n        text\n        == \"\"\"Some initial text\n\nacross multiple lines\n\"\"\"\n    )\n\n\ndef test_element_removal_full(client, live_server, measure_memory_usage, datastore_path):\n    \n\n    set_original_response(datastore_path=datastore_path)\n\n\n    # Add our URL to the import page\n    test_url = url_for(\"test_endpoint\", _external=True)\n    res = client.post(\n        url_for(\"imports.import_page\"), data={\"urls\": test_url}, follow_redirects=True\n    )\n    assert b\"1 Imported\" in res.data\n    wait_for_all_checks(client)\n\n    # Goto the edit page, add the filter data\n    # Not sure why \\r needs to be added - absent of the #changetext this is not necessary\n    subtractive_selectors_data = \"header\\r\\nfooter\\r\\nnav\\r\\n#changetext\"\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={\n            \"subtractive_selectors\": subtractive_selectors_data,\n            \"url\": test_url,\n            \"tags\": \"\",\n            \"headers\": \"\",\n            \"fetch_backend\": \"html_requests\",\n            \"time_between_check_use_default\": \"y\",\n        },\n        follow_redirects=True,\n    )\n    assert b\"Updated watch.\" in res.data\n    wait_for_all_checks(client)\n\n    # Check it saved\n    res = client.get(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n    )\n    assert bytes(subtractive_selectors_data.encode(\"utf-8\")) in res.data\n\n    # Trigger a check\n    res = client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    assert b'Queued 1 watch for rechecking.' in res.data\n\n    wait_for_all_checks(client)\n\n    # so that we set the state to 'has-unread-changes' after all the edits\n    client.get(url_for(\"ui.ui_diff.diff_history_page\", uuid=\"first\"))\n\n    #  Make a change to header/footer/nav\n    set_modified_response(datastore_path=datastore_path)\n\n    # Trigger a check\n    res = client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    assert b'Queued 1 watch for rechecking.' in res.data\n\n    # Give the thread time to pick it up\n    wait_for_all_checks(client)\n\n    # There should not be an unviewed change, as changes should be removed\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b\"unviewed\" not in res.data\n\n# Re #2752\ndef test_element_removal_nth_offset_no_shift(client, live_server, measure_memory_usage, datastore_path):\n\n    set_response_with_multiple_index(datastore_path=datastore_path)\n    subtractive_selectors_data = [\n### css style ###\n\"\"\"body > table > tr:nth-child(1) > th:nth-child(2)\nbody > table >  tr:nth-child(2) > td:nth-child(2)\nbody > table > tr:nth-child(3) > td:nth-child(2)\nbody > table > tr:nth-child(1) > th:nth-child(3)\nbody > table >  tr:nth-child(2) > td:nth-child(3)\nbody > table > tr:nth-child(3) > td:nth-child(3)\"\"\",\n### second type, xpath ###\n\"\"\"//body/table/tr[1]/th[2]\n//body/table/tr[2]/td[2]\n//body/table/tr[3]/td[2]\n//body/table/tr[1]/th[3]\n//body/table/tr[2]/td[3]\n//body/table/tr[3]/td[3]\"\"\"]\n    \n    test_url = url_for(\"test_endpoint\", _external=True)\n\n    for selector_list in subtractive_selectors_data:\n\n        delete_all_watches(client)\n\n        uuid = client.application.config.get('DATASTORE').add_watch(url=test_url, extras={\"subtractive_selectors\": selector_list.splitlines()})\n        client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n        wait_for_all_checks(client)\n\n        res = client.get(\n            url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"),\n            follow_redirects=True\n        )\n\n        # the filters above should have removed this but they never say to remove the \"emil\" column\n        assert b\"Tobias\" not in res.data\n        assert b\"Linus\" not in res.data\n        assert b\"Person 2\" not in res.data\n        assert b\"Person 3\" not in res.data\n        # First column should exist\n        assert b\"Emil\" in res.data\n\n"
  },
  {
    "path": "changedetectionio/tests/test_encoding.py",
    "content": "#!/usr/bin/env python3\n# coding=utf-8\n\nimport hashlib\nimport time\nfrom flask import url_for\nfrom .util import live_server_setup, wait_for_all_checks, extract_UUID_from_client\nimport pytest\nimport os\n\n\n\n\n\ndef test_surrogate_characters_in_content_are_sanitized():\n    \"\"\"Lone surrogates can appear in requests' r.text when a server returns malformed/mixed-encoding\n    content. Without sanitization, encoding to UTF-8 raises UnicodeEncodeError.\n    See: https://github.com/dgtlmoon/changedetection.io/issues/3952\n    \"\"\"\n    content_with_surrogate = 'Hello \\udcad World'\n\n    # Confirm the raw problem exists\n    with pytest.raises(UnicodeEncodeError):\n        content_with_surrogate.encode('utf-8')\n\n    # Our fix: sanitize after fetcher.run() in processors/base.py call_browser()\n    sanitized = content_with_surrogate.encode('utf-8', errors='replace').decode('utf-8')\n    assert 'Hello' in sanitized\n    assert 'World' in sanitized\n    assert '\\udcad' not in sanitized\n\n    # Checksum computation (processors/base.py get_raw_document_checksum) must not crash\n    hashlib.md5(sanitized.encode('utf-8')).hexdigest()\n\n\ndef test_utf8_content_without_charset_header(client, live_server, datastore_path):\n    \"\"\"Server returns UTF-8 content but no charset in Content-Type header.\n    chardet can misdetect such pages as UTF-7 (Python 3.14 then produces surrogates).\n    Our fix tries UTF-8 first before falling back to chardet.\n    See: https://github.com/dgtlmoon/changedetection.io/issues/3952\n    \"\"\"\n    from .util import write_test_file_and_sync\n    # UTF-8 encoded content with non-ASCII chars - no charset will be in the header\n    html = '
    Español
    Français
    日本語
    '\n    write_test_file_and_sync(os.path.join(datastore_path, \"endpoint-content.txt\"), html.encode('utf-8'), mode='wb')\n\n    test_url = url_for('test_endpoint', content_type=\"text/html\", _external=True)\n    client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    res = client.get(url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"), follow_redirects=True)\n    # Should decode correctly as UTF-8, not produce mojibake (Español) or replacement chars\n    assert 'Español'.encode('utf-8') in res.data\n    assert 'Français'.encode('utf-8') in res.data\n    assert '日本語'.encode('utf-8') in res.data\n\n\ndef test_shiftjis_with_meta_charset(client, live_server, datastore_path):\n    \"\"\"Server returns Shift-JIS content with no charset in HTTP header, but the HTML\n    declares . We should use the meta tag, not chardet.\n    Real-world case: https://github.com/dgtlmoon/changedetection.io/issues/3952\n    \"\"\"\n    from .util import write_test_file_and_sync\n    japanese_text = '日本語のページ'\n    html = f'
    {japanese_text}
    '\n    write_test_file_and_sync(os.path.join(datastore_path, \"endpoint-content.txt\"), html.encode('shift_jis'), mode='wb')\n\n    test_url = url_for('test_endpoint', content_type=\"text/html\", _external=True)\n    client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    res = client.get(url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"), follow_redirects=True)\n    assert japanese_text.encode('utf-8') in res.data\n\n\ndef set_html_response(datastore_path):\n    test_return_data = \"\"\"\n\n                         铸大国重器，挺制造脊梁，致力能源未来，赋能美好生活。\n                                  \n\n    \"\"\"\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n    return None\n\n\n# In the case the server does not issue a charset= or doesnt have content_type header set\ndef test_check_encoding_detection(client, live_server, measure_memory_usage, datastore_path):\n    set_html_response(datastore_path=datastore_path)\n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', content_type=\"text/html\", _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    # Give the thread time to pick it up\n    wait_for_all_checks(client)\n\n\n    # Content type recording worked\n    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))\n    assert live_server.app.config['DATASTORE'].data['watching'][uuid]['content-type'] == \"text/html\"\n\n    res = client.get(\n        url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"),\n        follow_redirects=True\n    )\n\n    # Should see the proper string\n    assert \"铸大国重\".encode('utf-8') in res.data\n    # Should not see the failed encoding\n    assert b'\\xc2\\xa7' not in res.data\n\n\n# In the case the server does not issue a charset= or doesnt have content_type header set\ndef test_check_encoding_detection_missing_content_type_header(client, live_server, measure_memory_usage, datastore_path):\n    set_html_response(datastore_path=datastore_path)\n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    wait_for_all_checks(client)\n\n    res = client.get(\n        url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"),\n        follow_redirects=True\n    )\n\n    # Should see the proper string\n    assert \"铸大国重\".encode('utf-8') in res.data\n    # Should not see the failed encoding\n    assert b'\\xc2\\xa7' not in res.data\n"
  },
  {
    "path": "changedetectionio/tests/test_errorhandling.py",
    "content": "#!/usr/bin/env python3\n\nimport time\nimport os\n\nfrom flask import url_for\nfrom .util import live_server_setup, wait_for_all_checks, delete_all_watches\n\n\n\n\ndef _runner_test_http_errors(client, live_server, http_code, expected_text, datastore_path):\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(\"Now you going to get a {} error code\\n\".format(http_code))\n\n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint',\n                       status_code=http_code,\n                       _external=True)\n\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    # Give the thread time to pick it up\n    wait_for_all_checks(client)\n\n    res = client.get(url_for(\"watchlist.index\"))\n    # no change\n    assert b'has-unread-changes' not in res.data\n    assert bytes(expected_text.encode('utf-8')) in res.data\n\n\n    # Error viewing tabs should appear\n    res = client.get(\n        url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"),\n        follow_redirects=True\n    )\n\n    assert b'Error Text' in res.data\n\n    # 'Error Screenshot' only when in playwright mode\n    #assert b'Error Screenshot' in res.data\n\n\n    delete_all_watches(client)\n\n\ndef test_http_error_handler(client, live_server, measure_memory_usage, datastore_path):\n    _runner_test_http_errors(client, live_server, 403, 'Access denied', datastore_path=datastore_path)\n    _runner_test_http_errors(client, live_server, 404, 'Page not found', datastore_path=datastore_path)\n    _runner_test_http_errors(client, live_server, 500, '(Internal server error) received', datastore_path=datastore_path)\n    _runner_test_http_errors(client, live_server, 400, 'Error - Request returned a HTTP error code 400', datastore_path=datastore_path)\n    delete_all_watches(client)\n\n# Just to be sure error text is properly handled\ndef test_DNS_errors(client, live_server, measure_memory_usage, datastore_path):\n\n    # Add our URL to the import page\n    res = client.post(\n        url_for(\"imports.import_page\"),\n        data={\"urls\": \"https://errorfuldomainthatnevereallyexists12356.com\"},\n        follow_redirects=True\n    )\n    assert b\"1 Imported\" in res.data\n    res = client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    # Give the thread time to pick it up\n    wait_for_all_checks(client)\n\n    res = client.get(url_for(\"watchlist.index\"))\n    found_name_resolution_error = (\n        b\"No address found\" in res.data or\n        b\"Name or service not known\" in res.data or\n        b\"nodename nor servname provided\" in res.data or\n        b\"Temporary failure in name resolution\" in res.data or\n        b\"Failed to establish a new connection\" in res.data or\n        b\"Connection error occurred\" in res.data\n    )\n    assert found_name_resolution_error\n    # Should always record that we tried\n    assert \"just now\".encode('utf-8') in res.data or 'seconds ago'.encode('utf-8') in res.data\n    delete_all_watches(client)\n\n# Re 1513\ndef test_low_level_errors_clear_correctly(client, live_server, measure_memory_usage, datastore_path):\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(\"
    Hello world
    \")\n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True)\n\n    res = client.post(\n        url_for(\"imports.import_page\"),\n        data={\"urls\": \"https://dfkjasdkfjaidjfsdajfksdajfksdjfDOESNTEXIST.com\"},\n        follow_redirects=True\n    )\n    assert b\"1 Imported\" in res.data\n    wait_for_all_checks(client)\n\n    # We should see the DNS error\n    res = client.get(url_for(\"watchlist.index\"))\n    found_name_resolution_error = (\n        b\"No address found\" in res.data or\n        b\"Name or service not known\" in res.data or\n        b\"nodename nor servname provided\" in res.data or\n        b\"Temporary failure in name resolution\" in res.data or\n        b\"Failed to establish a new connection\" in res.data or\n        b\"Connection error occurred\" in res.data\n    )\n    assert found_name_resolution_error\n\n    # Update with what should work\n    client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={\n            \"url\": test_url,\n            \"fetch_backend\": \"html_requests\",\n            \"time_between_check_use_default\": \"y\"},\n        follow_redirects=True\n    )\n\n    # Now the error should be gone\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"watchlist.index\"))\n    found_name_resolution_error = (\n        b\"No address found\" in res.data or\n        b\"Name or service not known\" in res.data or\n        b\"nodename nor servname provided\" in res.data or\n        b\"Temporary failure in name resolution\" in res.data or\n        b\"Failed to establish a new connection\" in res.data or\n        b\"Connection error occurred\" in res.data\n    )\n    assert not found_name_resolution_error\n\n    delete_all_watches(client)\n"
  },
  {
    "path": "changedetectionio/tests/test_extract_csv.py",
    "content": "#!/usr/bin/env python3\n\nimport time\nfrom flask import url_for\nfrom urllib.request import urlopen\nfrom .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks\nimport os\n\ndef test_check_extract_text_from_diff(client, live_server, measure_memory_usage, datastore_path):\n    import time\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(\"Now it's {} seconds since epoch, time flies!\".format(str(time.time())))\n\n   #  live_server_setup(live_server) # Setup on conftest per function\n\n    # Add our URL to the import page\n    res = client.post(\n        url_for(\"imports.import_page\"),\n        data={\"urls\": url_for('test_endpoint', _external=True)},\n        follow_redirects=True\n    )\n\n    assert b\"1 Imported\" in res.data\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"ui.ui_diff.diff_history_page_extract_GET\", uuid=\"first\"))\n    assert res.status_code == 200\n    assert b'extract_regex' in res.data\n\n    # Load in 5 different numbers/changes\n    last_date=\"\"\n    for n in range(5):\n        time.sleep(1)\n        # Give the thread time to pick it up\n        print(\"Bumping snapshot and checking.. \", n)\n        last_date = str(time.time())\n        with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n            f.write(\"Now it's {} seconds since epoch, time flies!\".format(last_date))\n\n        client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n        wait_for_all_checks(client)\n\n\n\n    res = client.post(\n        url_for(\"ui.ui_diff.diff_history_page_extract_POST\", uuid=\"first\"),\n        data={\"extract_regex\": \"Now it's ([0-9\\.]+)\",\n              \"extract_submit_button\": \"Extract as CSV\"},\n        follow_redirects=False\n    )\n\n    assert b'No matches found while scanning all of the watch history for that RegEx.' not in res.data\n    assert res.content_type == 'text/csv'\n\n    # Read the csv reply as stringio\n    from io import StringIO\n    import csv\n\n    f = StringIO(res.data.decode('utf-8'))\n    reader = csv.reader(f, delimiter=',')\n    output=[]\n\n    for row in reader:\n        output.append(row)\n\n    assert output[0][0] == 'Epoch seconds'\n\n    # Header line + 1 origin/first + 5 changes\n    assert(len(output) == 7)\n\n    # We expect to find the last bumped date in the changes in the last field of the spreadsheet\n    assert(output[6][2] == last_date)\n    # And nothing else, only that group () of the decimal and .\n    assert \"time flies\" not in output[6][2]\n"
  },
  {
    "path": "changedetectionio/tests/test_extract_regex.py",
    "content": "#!/usr/bin/env python3\n\nimport time\nfrom flask import url_for\nfrom .util import live_server_setup, wait_for_all_checks, delete_all_watches\nimport os\n\nfrom ..html_tools import *\n\n\ndef set_original_response(datastore_path):\n    test_return_data = \"\"\"\n       \n     Some initial text
    \n     
    Which is across multiple lines
    \n     
    \n     So let's see what happens.  
    \n     
    Some text thats the same
    \n     
    Some text that will change
         \n     \n     \n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n    return None\n\n\ndef set_modified_response(datastore_path):\n    test_return_data = \"\"\"\n       \n     Some initial text
    \n     
    which has this one new line
    \n     
    \n     So let's see what happens.  
    \n     
    Some text thats the same
    \n     
    Some text that did change ( 1000 online 
     80 guests
      2000 online )
    \n     
    SomeCase insensitive 3456
    \n     \n     \n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n\n    return None\n\n\ndef set_multiline_response(datastore_path):\n    test_return_data = \"\"\"\n       \n     \n     
    Something 
    \n        across 6 billion multiple
    \n        lines\n     
    \n     \n     
    aaand something lines
    \n     
    \n     
    and this should be
    \n     \n     \n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n\n    return None\n\n\n# def test_setup(client, live_server, measure_memory_usage, datastore_path):\n   #  live_server_setup(live_server) # Setup on conftest per function\n\ndef test_check_filter_multiline(client, live_server, measure_memory_usage, datastore_path):\n   ##  live_server_setup(live_server) # Setup on conftest per function\n    set_multiline_response(datastore_path=datastore_path)\n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    wait_for_all_checks(client)\n\n    # Goto the edit page, add our ignore text\n    # Add our URL to the import page\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={\"include_filters\": '',\n              # Test a regex and a plaintext\n              'extract_text': '/something.+?6 billion.+?lines/si\\r\\nand this should be',\n              \"url\": test_url,\n              \"tags\": \"\",\n              \"headers\": \"\",\n              'fetch_backend': \"html_requests\",\n              \"time_between_check_use_default\": \"y\"\n              },\n        follow_redirects=True\n    )\n\n    assert b\"Updated watch.\" in res.data\n    wait_for_all_checks(client)\n\n    res = client.get(url_for(\"watchlist.index\"))\n\n    # Issue 1828\n    assert b'not at the start of the expression' not in res.data\n\n    res = client.get(\n        url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"),\n        follow_redirects=True\n    )\n    # Plaintext that doesnt look like a regex should match also\n    assert b'and this should be' in res.data\n\n    assert b'Something' in res.data\n    assert b'across 6 billion multiple' in res.data\n    assert b'lines' in res.data\n\n    # but the last one, which also says 'lines' shouldnt be here (non-greedy match checking)\n    assert b'aaand something lines' not in res.data\n\ndef test_check_filter_and_regex_extract(client, live_server, measure_memory_usage, datastore_path):\n    \n    include_filters = \".changetext\"\n\n    set_original_response(datastore_path=datastore_path)\n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    # Give the thread time to pick it up\n    wait_for_all_checks(client)\n\n    # Goto the edit page, add our ignore text\n    # Add our URL to the import page\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={\"include_filters\": include_filters,\n              'extract_text': '/\\d+ online/\\r\\n/\\d+ guests/\\r\\n/somecase insensitive \\d+/i\\r\\n/somecase insensitive (345\\d)/i\\r\\n/issue1828.+?2022/i',\n              \"url\": test_url,\n              \"tags\": \"\",\n              \"headers\": \"\",\n              'fetch_backend': \"html_requests\",\n              \"time_between_check_use_default\": \"y\"\n              },\n        follow_redirects=True\n    )\n\n    assert b\"Updated watch.\" in res.data\n\n\n    # Give the thread time to pick it up\n    wait_for_all_checks(client)\n\n    res = client.get(url_for(\"watchlist.index\"))\n    #issue 1828\n    assert b'not at the start of the expression' not in res.data\n\n    #  Make a change\n    set_modified_response(datastore_path=datastore_path)\n\n    # Trigger a check\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    # Give the thread time to pick it up\n    wait_for_all_checks(client)\n\n    # It should have 'has-unread-changes' still\n    # Because it should be looking at only that 'sametext' id\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'has-unread-changes' in res.data\n\n    # Check HTML conversion detected and workd\n    res = client.get(\n        url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"),\n        follow_redirects=True\n    )\n\n    assert b'1000 online' in res.data\n\n    # All regex matching should be here\n    assert b'2000 online' in res.data\n\n    # Both regexs should be here\n    assert b'80 guests' in res.data\n\n    # Regex with flag handling should be here\n    assert b'SomeCase insensitive 3456' in res.data\n\n    # Singular group from /somecase insensitive (345\\d)/i\n    assert b'3456' in res.data\n\n    # Regex with multiline flag handling should be here\n\n    # Should not be here\n    assert b'Some text that did change' not in res.data\n\n\n\ndef test_regex_error_handling(client, live_server, measure_memory_usage, datastore_path):\n\n    \n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    time.sleep(0.2)\n    ### test regex error handling\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=uuid),\n        data={\"extract_text\": '/something bad\\d{3/XYZ',\n              \"url\": test_url,\n              \"fetch_backend\": \"html_requests\",\n              \"time_between_check_use_default\": \"y\"},\n        follow_redirects=True\n    )\n\n    assert b'is not a valid regular expression.' in res.data\n\n    delete_all_watches(client)\n"
  },
  {
    "path": "changedetectionio/tests/test_filter_exist_changes.py",
    "content": "#!/usr/bin/env python3\n\n# https://www.reddit.com/r/selfhosted/comments/wa89kp/comment/ii3a4g7/?context=3\nimport os\nimport time\nfrom flask import url_for\nfrom .util import set_original_response, live_server_setup, wait_for_notification_endpoint_output\nfrom changedetectionio.model import App\n\n\ndef set_response_without_filter(datastore_path):\n    test_return_data = \"\"\"\n       \n     Some initial text
    \n     
    Which is across multiple lines
    \n     
    \n     So let's see what happens.  
    \n     
    Some text thats the same
    \n     \n     \n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n    return None\n\n\ndef set_response_with_filter(datastore_path):\n    test_return_data = \"\"\"\n       \n     Some initial text
    \n     
    Which is across multiple lines
    \n     
    \n     So let's see what happens.  
    \n     
    Ticket now on sale!
    \n     \n     \n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n    return None\n\ndef test_filter_doesnt_exist_then_exists_should_get_notification(client, live_server, measure_memory_usage, datastore_path):\n#  Filter knowingly doesn't exist, like someone setting up a known filter to see if some cinema tickets are on sale again\n#  And the page has that filter available\n#  Then I should get a notification\n\n   #  live_server_setup(live_server) # Setup on conftest per function\n\n    # Give the endpoint time to spin up\n    time.sleep(1)\n    set_response_without_filter(datastore_path=datastore_path)\n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True)\n    res = client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": test_url, \"tags\": 'cinema'},\n        follow_redirects=True\n    )\n    assert b\"Watch added\" in res.data\n\n    # Give the thread time to pick up the first version\n    time.sleep(3)\n\n    # Goto the edit page, add our ignore text\n    # Add our URL to the import page\n    url = url_for('test_notification_endpoint', _external=True)\n    notification_url = url.replace('http', 'json')\n\n    print(\">>>> Notification URL: \" + notification_url)\n\n    # Just a regular notification setting, this will be used by the special 'filter not found' notification\n    notification_form_data = {\"notification_urls\": notification_url,\n                              \"notification_title\": \"New ChangeDetection.io Notification - {{watch_url}}\",\n                              \"notification_body\": \"BASE URL: {{base_url}}\\n\"\n                                                   \"Watch URL: {{watch_url}}\\n\"\n                                                   \"Watch UUID: {{watch_uuid}}\\n\"\n                                                   \"Watch title: {{watch_title}}\\n\"\n                                                   \"Watch tag: {{watch_tag}}\\n\"\n                                                   \"Preview: {{preview_url}}\\n\"\n                                                   \"Diff URL: {{diff_url}}\\n\"\n                                                   \"Snapshot: {{current_snapshot}}\\n\"\n                                                   \"Diff: {{diff}}\\n\"\n                                                   \"Diff Full: {{diff_full}}\\n\"\n                                                   \"Diff as Patch: {{diff_patch}}\\n\"\n                                                   \":-)\",\n                              \"notification_format\": 'text'}\n\n    notification_form_data.update({\n        \"url\": test_url,\n        \"tags\": \"my tag\",\n        \"title\": \"my title\",\n        \"headers\": \"\",\n        # preprended with extra filter that intentionally doesn't match any entry,\n        # notification should still be sent even if first filter does not match (PR#3516)\n        \"include_filters\": \".non-matching-selector\\n.ticket-available\",\n        \"fetch_backend\": \"html_requests\",\n        \"time_between_check_use_default\": \"y\"})\n\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data=notification_form_data,\n        follow_redirects=True\n    )\n    assert b\"Updated watch.\" in res.data\n    wait_for_notification_endpoint_output(datastore_path=datastore_path)\n\n    # Shouldn't exist, shouldn't have fired\n    assert not os.path.isfile(os.path.join(datastore_path, \"notification.txt\"))\n    # Now the filter should exist\n    set_response_with_filter(datastore_path=datastore_path)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    wait_for_notification_endpoint_output(datastore_path=datastore_path)\n\n    assert os.path.isfile(os.path.join(datastore_path, \"notification.txt\"))\n\n    with open(os.path.join(datastore_path, \"notification.txt\"), 'r') as f:\n        notification = f.read()\n\n    assert 'Ticket now on sale' in notification\n    os.unlink(os.path.join(datastore_path, \"notification.txt\"))\n"
  },
  {
    "path": "changedetectionio/tests/test_filter_failure_notification.py",
    "content": "import os\nimport time\nfrom flask import url_for\nfrom .util import set_original_response, wait_for_all_checks, wait_for_notification_endpoint_output, delete_all_watches\nfrom ..notification import valid_notification_formats\n\n\ndef set_response_with_filter(datastore_path):\n    test_return_data = \"\"\"\n       \n     Some initial text
    \n     
    Which is across multiple lines
    \n     
    \n     So let's see what happens.  
    \n     
    Some text thats the same
    \n     \n     \n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n    return None\n\ndef run_filter_test(client, live_server, content_filter, app_notification_format, datastore_path):\n\n    # Response WITHOUT the filter ID element\n    set_original_response(datastore_path=datastore_path)\n    live_server.app.config['DATASTORE'].data['settings']['application']['notification_format'] = app_notification_format\n\n    # Goto the edit page, add our ignore text\n    notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'post')\n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True)\n\n    # cleanup for the next\n    client.get(\n        url_for(\"ui.form_delete\", uuid=\"all\"),\n        follow_redirects=True\n    )\n    notification_file = os.path.join(datastore_path, \"notification.txt\")\n    if os.path.isfile(notification_file):\n        os.unlink(notification_file)\n\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    res = client.get(url_for(\"watchlist.index\"))\n\n    assert b'No web page change detection watches configured' not in res.data\n\n\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))\n\n    assert live_server.app.config['DATASTORE'].data['watching'][uuid]['consecutive_filter_failures'] == 0, \"No filter = No filter failure\"\n\n    watch_data = {\"notification_urls\": notification_url,\n                  \"notification_title\": \"New ChangeDetection.io Notification - {{watch_url}}\",\n                  \"notification_body\": \"BASE URL: {{base_url}}\\n\"\n                                       \"Watch URL: {{watch_url}}\\n\"\n                                       \"Watch UUID: {{watch_uuid}}\\n\"\n                                       \"Watch title: {{watch_title}}\\n\"\n                                       \"Watch tag: {{watch_tag}}\\n\"\n                                       \"Preview: {{preview_url}}\\n\"\n                                       \"Diff URL: {{diff_url}}\\n\"\n                                       \"Snapshot: {{current_snapshot}}\\n\"\n                                       \"Diff: {{diff}}\\n\"\n                                       \"Diff Full: {{diff_full}}\\n\"\n                                       \"Diff as Patch: {{diff_patch}}\\n\"\n                                       \":-)\",\n                  \"notification_format\": 'text',\n                  \"fetch_backend\": \"html_requests\",\n                  \"filter_failure_notification_send\": 'y',\n                  \"time_between_check_use_default\": \"y\",\n                  \"headers\": \"\",\n                  \"tags\": \"my tag\",\n                  \"title\": \"my title 123\",\n                  \"time_between_check-hours\": 5,  # So that the queue runner doesnt also put it in\n                  \"url\": test_url,\n                  }\n\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=uuid),\n        data=watch_data,\n        follow_redirects=True\n    )\n\n    assert b\"Updated watch.\" in res.data\n    wait_for_all_checks(client)\n    assert live_server.app.config['DATASTORE'].data['watching'][uuid]['consecutive_filter_failures'] == 0, \"No filter = No filter failure\"\n\n    # Now add a filter, because recheck hours == 5, ONLY pressing of the [edit] or [recheck all] should trigger\n    watch_data['include_filters'] = content_filter\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=uuid),\n        data=watch_data,\n        follow_redirects=True\n    )\n    assert b\"Updated watch.\" in res.data\n\n    # It should have checked once so far and given this error (because we hit SAVE)\n\n    wait_for_all_checks(client)\n    assert not os.path.isfile(notification_file)\n\n    # Hitting [save] would have triggered a recheck, and we have a filter, so this would be ONE failure\n    assert live_server.app.config['DATASTORE'].data['watching'][uuid]['consecutive_filter_failures'] == 1, \"Should have been checked once\"\n\n    # recheck it up to just before the threshold, including the fact that in the previous POST it would have rechecked (and incremented)\n    # Add 4 more checks\n    checked = 0\n    ATTEMPT_THRESHOLD_SETTING = live_server.app.config['DATASTORE'].data['settings']['application'].get('filter_failure_notification_threshold_attempts', 0)\n    for i in range(0, ATTEMPT_THRESHOLD_SETTING - 2):\n        checked += 1\n        client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n        wait_for_all_checks(client)\n        res = client.get(url_for(\"watchlist.index\"))\n        assert b'Warning, no filters were found' in res.data\n        assert not os.path.isfile(notification_file)\n        time.sleep(2)\n        wait_for_all_checks(client)\n\n    wait_for_all_checks(client)\n    assert live_server.app.config['DATASTORE'].data['watching'][uuid]['consecutive_filter_failures'] == 5\n\n    time.sleep(2)\n    # One more check should trigger the _FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT threshold\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n    wait_for_notification_endpoint_output(datastore_path=datastore_path)\n\n    # Now it should exist and contain our \"filter not found\" alert\n    assert os.path.isfile(notification_file)\n    with open(notification_file, 'r') as f:\n        notification = f.read()\n\n    assert 'Your configured CSS/xPath filters' in notification\n\n\n    # Text (or HTML conversion) markup to make the notifications a little nicer should have worked\n    if app_notification_format.startswith('html'):\n        # apprise should have used sax-escape (' instead of ", \" etc), lets check it worked\n\n        from apprise.conversion import convert_between\n        from apprise.common import NotifyFormat\n        escaped_filter = convert_between(NotifyFormat.TEXT, NotifyFormat.HTML, content_filter)\n\n        assert escaped_filter in notification or escaped_filter.replace('"', '"') in notification\n        assert 'a href=\"' in notification # Quotes should still be there so the link works\n\n    else:\n        assert 'a href' not in notification\n        assert content_filter in notification\n\n    # Remove it and prove that it doesn't trigger when not expected\n    # It should register a change, but no 'filter not found'\n    os.unlink(notification_file)\n    set_response_with_filter(datastore_path)\n\n    # Try several times, it should NOT have 'filter not found'\n    for i in range(0, ATTEMPT_THRESHOLD_SETTING + 2):\n        client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n        wait_for_all_checks(client)\n\n    wait_for_notification_endpoint_output(datastore_path=datastore_path)\n    # It should have sent a notification, but..\n    assert os.path.isfile(notification_file)\n    # but it should not contain the info about a failed filter (because there was none in this case)\n    with open(notification_file, 'r') as f:\n        notification = f.read()\n    assert not 'CSS/xPath filter was not present in the page' in notification\n\n    # Re #1247 - All tokens got replaced correctly in the notification\n    assert uuid in notification\n\n    # cleanup for the next\n    client.get(\n        url_for(\"ui.form_delete\", uuid=\"all\"),\n        follow_redirects=True\n    )\n    os.unlink(notification_file)\n    delete_all_watches(client)\n\n\ndef test_check_include_filters_failure_notification(client, live_server, measure_memory_usage, datastore_path):\n    #   #  live_server_setup(live_server) # Setup on conftest per function\n    run_filter_test(client=client, live_server=live_server, content_filter='#nope-doesnt-exist', app_notification_format=valid_notification_formats.get('htmlcolor'), datastore_path=datastore_path)\n    # Check markup send conversion didnt affect plaintext preference\n    run_filter_test(client=client, live_server=live_server, content_filter='#nope-doesnt-exist', app_notification_format=valid_notification_formats.get('text'), datastore_path=datastore_path)\n    delete_all_watches(client)\n\ndef test_check_xpath_filter_failure_notification(client, live_server, measure_memory_usage, datastore_path):\n    #   #  live_server_setup(live_server) # Setup on conftest per function\n    run_filter_test(client=client, live_server=live_server, content_filter='//*[@id=\"nope-doesnt-exist\"]', app_notification_format=valid_notification_formats.get('htmlcolor'), datastore_path=datastore_path)\n    delete_all_watches(client)\n\n# Test that notification is never sent\n\ndef test_basic_markup_from_text(client, live_server, measure_memory_usage, datastore_path):\n    # Test the notification error templates convert to HTML if needed (link activate)\n    from ..notification.handler import markup_text_links_to_html\n    x = markup_text_links_to_html(\"hello https://google.com\")\n    assert 'a href' in x\n    delete_all_watches(client)\n"
  },
  {
    "path": "changedetectionio/tests/test_group.py",
    "content": "#!/usr/bin/env python3\n\nimport time\nfrom flask import url_for\nfrom .util import live_server_setup, wait_for_all_checks, extract_rss_token_from_UI, get_UUID_for_tag_name, extract_UUID_from_client, delete_all_watches\nimport os\n\nfrom ..store import ChangeDetectionStore\n\n\n# def test_setup(client, live_server, measure_memory_usage, datastore_path):\n   #  live_server_setup(live_server) # Setup on conftest per function\n\ndef set_original_response(datastore_path):\n    test_return_data = \"\"\"\n       \n     Some initial text
    \n     
    Should be only this
    \n     
    \n     
    And never this
    \n     \n     \n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n    return None\n\ndef set_modified_response(datastore_path):\n    test_return_data = \"\"\"\n       \n     Some initial text
    \n     
    Should be REALLY only this
    \n     
    \n     
    And never this
    \n     \n     \n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n    return None\n\ndef test_setup_group_tag(client, live_server, measure_memory_usage, datastore_path):\n    \n    set_original_response(datastore_path=datastore_path)\n\n    # Add a tag with some config, import a tag and it should roughly work\n    res = client.post(\n        url_for(\"tags.form_tag_add\"),\n        data={\"name\": \"test-tag\"},\n        follow_redirects=True\n    )\n    assert b\"Tag added\" in res.data\n    assert b\"test-tag\" in res.data\n\n    res = client.post(\n        url_for(\"tags.form_tag_edit_submit\", uuid=\"first\"),\n        data={\"name\": \"test-tag\",\n              \"include_filters\": '#only-this',\n              \"subtractive_selectors\": '#not-this'},\n        follow_redirects=True\n    )\n    assert b\"Updated\" in res.data\n    tag_uuid = get_UUID_for_tag_name(client, name=\"test-tag\")\n    res = client.get(\n        url_for(\"tags.form_tag_edit\", uuid=\"first\")\n    )\n    assert b\"#only-this\" in res.data\n    assert b\"#not-this\" in res.data\n\n    # Tag should be setup and ready, now add a watch\n\n    test_url = url_for('test_endpoint', _external=True)\n    res = client.post(\n        url_for(\"imports.import_page\"),\n        data={\"urls\": test_url + \"?first-imported=1 test-tag, extra-import-tag\"},\n        follow_redirects=True\n    )\n    assert b\"1 Imported\" in res.data\n\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'import-tag' in res.data\n    assert b'extra-import-tag' in res.data\n\n    res = client.get(\n        url_for(\"tags.tags_overview_page\"),\n        follow_redirects=True\n    )\n    assert b'import-tag' in res.data\n    assert b'extra-import-tag' in res.data\n\n    wait_for_all_checks(client)\n\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'Warning, no filters were found' not in res.data\n\n    res = client.get(\n        url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"),\n        follow_redirects=True\n    )\n    assert b'Should be only this' in res.data\n    assert b'And never this' not in res.data\n\n    res = client.get(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        follow_redirects=True\n    )\n    # 2307 the UI notice should appear in the placeholder\n    assert b'WARNING: Watch has tag/groups set with special filters' in res.data\n\n    # RSS Group tag filter\n    # An extra one that should be excluded\n    res = client.post(\n        url_for(\"imports.import_page\"),\n        data={\"urls\": test_url + \"?should-be-excluded=1 some-tag\"},\n        follow_redirects=True\n    )\n    assert b\"1 Imported\" in res.data\n    wait_for_all_checks(client)\n    set_modified_response(datastore_path=datastore_path)\n    res = client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n    rss_token = extract_rss_token_from_UI(client)\n    res = client.get(\n        url_for(\"rss.feed\", token=rss_token, tag=\"extra-import-tag\", _external=True),\n        follow_redirects=True\n    )\n    assert b\"should-be-excluded\" not in res.data\n    assert res.status_code == 200\n    assert b\"first-imported=1\" in res.data\n    delete_all_watches(client)\n\ndef test_tag_import_singular(client, live_server, measure_memory_usage, datastore_path):\n    \n\n    test_url = url_for('test_endpoint', _external=True)\n    res = client.post(\n        url_for(\"imports.import_page\"),\n        data={\"urls\": test_url + \" test-tag, test-tag\\r\\n\"+ test_url + \"?x=1 test-tag, test-tag\\r\\n\"},\n        follow_redirects=True\n    )\n    assert b\"2 Imported\" in res.data\n\n    res = client.get(\n        url_for(\"tags.tags_overview_page\"),\n        follow_redirects=True\n    )\n    # Should be only 1 tag because they both had the same\n    assert len(live_server.app.config['DATASTORE'].data['settings']['application'].get('tags')) ==1\n\n    delete_all_watches(client)\n\ndef test_tag_add_in_ui(client, live_server, measure_memory_usage, datastore_path):\n    \n#\n    res = client.post(\n        url_for(\"tags.form_tag_add\"),\n        data={\"name\": \"new-test-tag\"},\n        follow_redirects=True\n    )\n    assert b\"Tag added\" in res.data\n    assert b\"new-test-tag\" in res.data\n\n    res = client.get(url_for(\"tags.delete_all\"), follow_redirects=True)\n    assert b'All tags deleted' in res.data\n\n    delete_all_watches(client)\n\ndef test_group_tag_notification(client, live_server, measure_memory_usage, datastore_path):\n    delete_all_watches(client)\n\n    set_original_response(datastore_path=datastore_path)\n\n    test_url = url_for('test_endpoint', _external=True)\n    res = client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": test_url, \"tags\": 'test-tag, other-tag'},\n        follow_redirects=True\n    )\n\n    assert b\"Watch added\" in res.data\n\n    notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')\n    notification_form_data = {\"notification_urls\": notification_url,\n                              \"notification_title\": \"New GROUP TAG ChangeDetection.io Notification - {{watch_url}}\",\n                              \"notification_body\": \"BASE URL: {{base_url}}\\n\"\n                                                   \"Watch URL: {{watch_url}}\\n\"\n                                                   \"Watch UUID: {{watch_uuid}}\\n\"\n                                                   \"Watch title: {{watch_title}}\\n\"\n                                                   \"Watch tag: {{watch_tag}}\\n\"\n                                                   \"Preview: {{preview_url}}\\n\"\n                                                   \"Diff URL: {{diff_url}}\\n\"\n                                                   \"Snapshot: {{current_snapshot}}\\n\"\n                                                   \"Diff: {{diff}}\\n\"\n                                                   \"Diff Added: {{diff_added}}\\n\"\n                                                   \"Diff Removed: {{diff_removed}}\\n\"\n                                                   \"Diff Full: {{diff_full}}\\n\"\n                                                   \"Diff as Patch: {{diff_patch}}\\n\"\n                                                   \":-)\",\n                              \"notification_screenshot\": True,\n                              \"notification_format\": 'text',\n                              \"title\": \"test-tag\"}\n\n    res = client.post(\n        url_for(\"tags.form_tag_edit_submit\", uuid=get_UUID_for_tag_name(client, name=\"test-tag\")),\n        data=notification_form_data,\n        follow_redirects=True\n    )\n    assert b\"Updated\" in res.data\n\n    wait_for_all_checks(client)\n\n    set_modified_response(datastore_path=datastore_path)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    time.sleep(3)\n\n    assert os.path.isfile(os.path.join(datastore_path, \"notification.txt\"))\n\n    # Verify what was sent as a notification, this file should exist\n    with open(os.path.join(datastore_path, \"notification.txt\"), \"r\") as f:\n        notification_submission = f.read()\n    os.unlink(os.path.join(datastore_path, \"notification.txt\"))\n\n    # Did we see the URL that had a change, in the notification?\n    # Diff was correctly executed\n    assert test_url in notification_submission\n    assert ':-)' in notification_submission\n    assert \"Diff Full: Some initial text\" in notification_submission\n    assert \"New GROUP TAG ChangeDetection.io\" in notification_submission\n    assert \"test-tag\" in notification_submission\n    assert \"other-tag\" in notification_submission\n\n    #@todo Test that multiple notifications fired\n    #@todo Test that each of multiple notifications with different settings\n    delete_all_watches(client)\n\ndef test_limit_tag_ui(client, live_server, measure_memory_usage, datastore_path):\n\n    test_url = url_for('test_random_content_endpoint', _external=True)\n\n    # A space can label the tag, only the first one will have a tag\n    client.post(\n        url_for(\"imports.import_page\"),\n        data={\"urls\": f\"{test_url} test-tag\\r\\n{test_url}\"},\n        follow_redirects=True\n    )\n    tag_uuid = get_UUID_for_tag_name(client, name=\"test-tag\")\n    assert tag_uuid\n\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'test-tag' in res.data\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    # Should be both unviewed\n    res = client.get(url_for(\"watchlist.index\"))\n    assert res.data.count(b' unviewed ') == 2\n\n\n    # Now we recheck only the tag\n    client.get(url_for('ui.mark_all_viewed', tag=tag_uuid), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    # Should be only 1 unviewed\n    res = client.get(url_for(\"watchlist.index\"))\n    assert res.data.count(b' unviewed ') == 1\n\n\n    delete_all_watches(client)\n    res = client.get(url_for(\"tags.delete_all\"), follow_redirects=True)\n    assert b'All tags deleted' in res.data\n\ndef test_clone_tag_on_import(client, live_server, measure_memory_usage, datastore_path):\n    \n    test_url = url_for('test_endpoint', _external=True)\n    res = client.post(\n        url_for(\"imports.import_page\"),\n        data={\"urls\": test_url + \" test-tag, another-tag\\r\\n\"},\n        follow_redirects=True\n    )\n\n    assert b\"1 Imported\" in res.data\n\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'test-tag' in res.data\n    assert b'another-tag' in res.data\n\n    watch_uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))\n    res = client.get(url_for(\"ui.form_clone\", uuid=watch_uuid), follow_redirects=True)\n\n    assert b'Cloned' in res.data\n    res = client.get(url_for(\"watchlist.index\"))\n    # 2 times plus the top link to tag\n    assert res.data.count(b'test-tag') == 3\n    assert res.data.count(b'another-tag') == 3\n    delete_all_watches(client)\n\ndef test_clone_tag_on_quickwatchform_add(client, live_server, measure_memory_usage, datastore_path):\n    \n\n    test_url = url_for('test_endpoint', _external=True)\n\n    res = client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": test_url, \"tags\": ' test-tag, another-tag      '},\n        follow_redirects=True\n    )\n\n    assert b\"Watch added\" in res.data\n\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'test-tag' in res.data\n    assert b'another-tag' in res.data\n\n    watch_uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))\n    res = client.get(url_for(\"ui.form_clone\", uuid=watch_uuid), follow_redirects=True)\n    assert b'Cloned' in res.data\n\n    res = client.get(url_for(\"watchlist.index\"))\n    # 2 times plus the top link to tag\n    assert res.data.count(b'test-tag') == 3\n    assert res.data.count(b'another-tag') == 3\n    delete_all_watches(client)\n\n    res = client.get(url_for(\"tags.delete_all\"), follow_redirects=True)\n    assert b'All tags deleted' in res.data\n\ndef test_order_of_filters_tag_filter_and_watch_filter(client, live_server, measure_memory_usage, datastore_path):\n\n    # Add a tag with some config, import a tag and it should roughly work\n    res = client.post(\n        url_for(\"tags.form_tag_add\"),\n        data={\"name\": \"test-tag-keep-order\"},\n        follow_redirects=True\n    )\n    assert b\"Tag added\" in res.data\n    assert b\"test-tag-keep-order\" in res.data\n    tag_filters = [\n            '#only-this', # duplicated filters\n            '#only-this',\n            '#only-this',\n            '#only-this',\n            ]\n\n    res = client.post(\n        url_for(\"tags.form_tag_edit_submit\", uuid=\"first\"),\n        data={\"name\": \"test-tag-keep-order\",\n              \"include_filters\": '\\n'.join(tag_filters) },\n        follow_redirects=True\n    )\n    assert b\"Updated\" in res.data\n    tag_uuid = get_UUID_for_tag_name(client, name=\"test-tag-keep-order\")\n    res = client.get(\n        url_for(\"tags.form_tag_edit\", uuid=\"first\")\n    )\n    assert b\"#only-this\" in res.data\n\n\n    d = \"\"\"\n       \n     Some initial text
    \n     
    And 1 this
    \n     
    \n     
    And 2 this
    \n     
    And 3 this
    \n     
    And 4 this
    \n     
    And 5 this
    \n     
    And 6 this
    \n     
    And 7 this
    \n     
    And 8 this
    \n     
    And 9 this
    \n     
    And 10 this
    \n     
    And 11 this
    \n     
    And 12 this
    \n     
    And 13 this
    \n     
    And 14 this
    \n     
    And 15 this
    \n     \n     \n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(d)\n\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    filters = [\n            '/html/body/p[3]',\n            '/html/body/p[4]',\n            '/html/body/p[5]',\n            '/html/body/p[6]',\n            '/html/body/p[7]',\n            '/html/body/p[8]',\n            '/html/body/p[9]',\n            '/html/body/p[10]',\n            '/html/body/p[11]',\n            '/html/body/p[12]',\n            '/html/body/p[13]', # duplicated tags\n            '/html/body/p[13]',\n            '/html/body/p[13]',\n            '/html/body/p[13]',\n            '/html/body/p[13]',\n            '/html/body/p[14]',\n            ]\n\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={\"include_filters\": '\\n'.join(filters),\n            \"url\": test_url,\n            \"tags\": \"test-tag-keep-order\",\n            \"headers\": \"\",\n            'fetch_backend': \"html_requests\",\n            \"time_between_check_use_default\": \"y\"},\n        follow_redirects=True\n    )\n    assert b\"Updated watch.\" in res.data\n    wait_for_all_checks(client)\n\n    res = client.get(\n        url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"),\n        follow_redirects=True\n    )\n\n    assert b\"And 1 this\" in res.data  # test-tag-keep-order\n\n    a_tag_filter_check = b'And 1 this' #'#only-this' of tag_filters\n    # check there is no duplication of tag_filters\n    assert res.data.count(a_tag_filter_check) == 1, f\"duplicated filters didn't removed {res.data.count(a_tag_filter_check)} of {a_tag_filter_check} in {res.data=}\"\n\n    a_filter_check = b\"And 13 this\" # '/html/body/p[13]'\n    # check there is no duplication of filters\n    assert res.data.count(a_filter_check) == 1, f\"duplicated filters didn't removed. {res.data.count(a_filter_check)} of {a_filter_check} in {res.data=}\"\n\n    a_filter_check_not_include = b\"And 2 this\" # '/html/body/p[2]'\n    assert a_filter_check_not_include not in res.data\n\n    checklist = [\n            b\"And 3 this\",\n            b\"And 4 this\",\n            b\"And 5 this\",\n            b\"And 6 this\",\n            b\"And 7 this\",\n            b\"And 8 this\",\n            b\"And 9 this\",\n            b\"And 10 this\",\n            b\"And 11 this\",\n            b\"And 12 this\",\n            b\"And 13 this\",\n            b\"And 14 this\",\n            b\"And 1 this\", # result of filter from tag.\n            ]\n    # check whether everything a user requested is there\n    for test in checklist:\n        assert test in res.data\n\n    # check whether everything a user requested is in order of filters.\n    n = 0\n    for test in checklist:\n        t_index = res.data[n:].find(test)\n        # if the text is not searched, return -1.\n        assert t_index >= 0, f\"\"\"failed because {test=} not in {res.data[n:]=}\n#####################\nLooks like some feature changed the order of result of filters.\n#####################\nthe {test} appeared before. {test in res.data[:n]=}\n{res.data[:n]=}\n        \"\"\"\n        n += t_index + len(test)\n\n    delete_all_watches(client)\n\n\ndef test_tag_json_persistence(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test that tags are saved to individual tag.json files and loaded correctly.\n\n    This test verifies the update_27 tag storage refactoring:\n    - Tags are saved to {uuid}/tag.json files\n    - Tags persist across datastore restarts\n    - Tag edits write to tag.json\n    - Tag deletion removes tag.json file\n    \"\"\"\n    import json\n\n    datastore = client.application.config.get('DATASTORE')\n\n    # 1. Create a tag\n    res = client.post(\n        url_for(\"tags.form_tag_add\"),\n        data={\"name\": \"persistence-test-tag\"},\n        follow_redirects=True\n    )\n    assert b\"Tag added\" in res.data\n\n    tag_uuid = get_UUID_for_tag_name(client, name=\"persistence-test-tag\")\n    assert tag_uuid, \"Tag UUID should exist\"\n\n    # 2. Verify tag.json file was created\n    tag_json_path = os.path.join(datastore_path, tag_uuid, \"tag.json\")\n    assert os.path.exists(tag_json_path), f\"tag.json should exist at {tag_json_path}\"\n\n    # 3. Verify tag.json contains correct data\n    with open(tag_json_path, 'r') as f:\n        tag_data = json.load(f)\n    assert tag_data['title'] == 'persistence-test-tag'\n    assert tag_data['uuid'] == tag_uuid\n    assert 'date_created' in tag_data\n\n    # 4. Edit the tag\n    res = client.post(\n        url_for(\"tags.form_tag_edit_submit\", uuid=tag_uuid),\n        data={\n            \"name\": \"persistence-test-tag\",\n            \"notification_muted\": True,\n            \"include_filters\": '#test-filter'\n        },\n        follow_redirects=True\n    )\n    assert b\"Updated\" in res.data\n\n    # 5. Verify tag.json was updated\n    with open(tag_json_path, 'r') as f:\n        tag_data = json.load(f)\n    assert tag_data['notification_muted'] == True\n    assert '#test-filter' in tag_data.get('include_filters', [])\n\n    # 5a. Verify tag is NOT in changedetection.json (tags should be in tag.json only)\n    changedetection_json_path = os.path.join(datastore_path, \"changedetection.json\")\n    with open(changedetection_json_path, 'r') as f:\n        settings_data = json.load(f)\n    # Tags dict should be empty in settings (all tags are in individual files)\n    assert settings_data['settings']['application']['tags'] == {}, \\\n        \"Tags should NOT be saved to changedetection.json (should be empty dict)\"\n\n    # 6. Simulate restart - reload datastore\n    datastore2 = ChangeDetectionStore(datastore_path=datastore_path, include_default_watches=False, version_tag='test')\n\n    # 7. Verify tag was loaded from tag.json\n    assert tag_uuid in datastore2.data['settings']['application']['tags']\n    loaded_tag = datastore2.data['settings']['application']['tags'][tag_uuid]\n    assert loaded_tag['title'] == 'persistence-test-tag'\n    assert loaded_tag['notification_muted'] == True\n    assert '#test-filter' in loaded_tag.get('include_filters', [])\n\n    # 8. Delete the tag via API\n    res = client.get(url_for(\"tags.delete\", uuid=tag_uuid), follow_redirects=True)\n    assert b\"Tag deleted\" in res.data\n\n    # 9. Verify tag.json file was deleted\n    assert not os.path.exists(tag_json_path), f\"tag.json should be deleted at {tag_json_path}\"\n\n    # 10. Verify tag is removed from settings\n    assert tag_uuid not in datastore.data['settings']['application']['tags']\n\n    delete_all_watches(client)\n\n\ndef test_tag_json_migration_update_27(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test that update_27 migration correctly moves tags to individual files.\n\n    This simulates a pre-update_27 datastore and verifies migration works.\n    \"\"\"\n    import json\n\n    # 1. Create multiple tags\n    tag_names = ['migration-tag-1', 'migration-tag-2', 'migration-tag-3']\n    tag_uuids = []\n\n    for tag_name in tag_names:\n        res = client.post(\n            url_for(\"tags.form_tag_add\"),\n            data={\"name\": tag_name},\n            follow_redirects=True\n        )\n        assert b\"Tag added\" in res.data\n        tag_uuid = get_UUID_for_tag_name(client, name=tag_name)\n        tag_uuids.append(tag_uuid)\n\n    # 2. Verify all tag.json files exist (update_27 already ran during add_tag)\n    for tag_uuid in tag_uuids:\n        tag_json_path = os.path.join(datastore_path, tag_uuid, \"tag.json\")\n        assert os.path.exists(tag_json_path), f\"tag.json should exist for {tag_uuid}\"\n\n    # 2a. Verify tags are NOT in changedetection.json\n    changedetection_json_path = os.path.join(datastore_path, \"changedetection.json\")\n    with open(changedetection_json_path, 'r') as f:\n        settings_data = json.load(f)\n    assert settings_data['settings']['application']['tags'] == {}, \\\n        \"Tags should NOT be in changedetection.json after migration\"\n\n    # 3. Simulate restart\n    datastore2 = ChangeDetectionStore(datastore_path=datastore_path, include_default_watches=False, version_tag='test')\n\n    # 4. Verify all tags loaded from tag.json files\n    for idx, tag_uuid in enumerate(tag_uuids):\n        assert tag_uuid in datastore2.data['settings']['application']['tags']\n        loaded_tag = datastore2.data['settings']['application']['tags'][tag_uuid]\n        assert loaded_tag['title'] == tag_names[idx]\n\n    # Cleanup\n    res = client.get(url_for(\"tags.delete_all\"), follow_redirects=True)\n    assert b'All tags deleted' in res.data\n\n    # Verify all tag.json files were deleted\n    for tag_uuid in tag_uuids:\n        tag_json_path = os.path.join(datastore_path, tag_uuid, \"tag.json\")\n        assert not os.path.exists(tag_json_path), f\"tag.json should be deleted for {tag_uuid}\"\n\n    delete_all_watches(client)\n"
  },
  {
    "path": "changedetectionio/tests/test_history_consistency.py",
    "content": "#!/usr/bin/env python3\n\nimport time\nimport os\nimport json\nfrom flask import url_for\nfrom loguru import logger\nfrom .. import strtobool\nfrom .util import wait_for_all_checks, delete_all_watches\nimport brotli\n\n\ndef test_consistent_history(client, live_server, measure_memory_usage, datastore_path):\n\n    uuids = set()\n    sys_fetch_workers = int(os.getenv(\"FETCH_WORKERS\", 10))\n    workers = range(1, sys_fetch_workers)\n    now = time.time()\n\n    for one in workers:\n        if strtobool(os.getenv(\"TEST_WITH_BROTLI\")):\n            # A very long string that WILL trigger Brotli compression of the snapshot\n            # BROTLI_COMPRESS_SIZE_THRESHOLD should be set to say 200\n            from ..model.Watch import BROTLI_COMPRESS_SIZE_THRESHOLD\n            content = str(one) + \"x\" + str(one) * (BROTLI_COMPRESS_SIZE_THRESHOLD + 10)\n        else:\n            # Just enough to test datastore\n            content = str(one)+'x'\n\n        test_url = url_for('test_endpoint', content_type=\"text/html\", content=content, _external=True)\n        uuids.add(client.application.config.get('DATASTORE').add_watch(url=test_url, extras={'title': str(one)}))\n\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    wait_for_all_checks(client)\n    duration = time.time() - now\n    per_worker = duration/sys_fetch_workers\n    if sys_fetch_workers < 20:\n        per_worker_threshold=0.6\n    elif sys_fetch_workers < 50:\n        per_worker_threshold = 0.8\n    else:\n        per_worker_threshold = 1.5\n\n    logger.debug(f\"All fetched in {duration:.2f}s, {per_worker}s per worker\")\n    # Problematic on github\n    #assert per_worker < per_worker_threshold, f\"If concurrency is working good, no blocking async problems, each worker ({sys_fetch_workers} workers) should have done his job in under {per_worker_threshold}s, got {per_worker:.2f}s per worker, total duration was {duration:.2f}s\"\n\n    # Essentially just triggers the DB write/update\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\"application-empty_pages_are_a_change\": \"\",\n              \"requests-time_between_check-minutes\": 180,\n              'application-fetch_backend': \"html_requests\"},\n        follow_redirects=True\n    )\n    assert b\"Settings updated.\" in res.data\n\n    # Wait for the sync DB save to happen\n    time.sleep(2)\n\n    # Check which format is being used\n    datastore_path = live_server.app.config['DATASTORE'].datastore_path\n    changedetection_json = os.path.join(datastore_path, 'changedetection.json')\n    url_watches_json = os.path.join(datastore_path, 'url-watches.json')\n\n    json_obj = {'watching': {}}\n\n    if os.path.exists(changedetection_json):\n        # New format: individual watch.json files\n        logger.info(\"Testing with new format (changedetection.json + individual watch.json)\")\n\n        # Load each watch.json file\n        for uuid in live_server.app.config['DATASTORE'].data['watching'].keys():\n            watch_json_file = os.path.join(datastore_path, uuid, 'watch.json')\n            assert os.path.isfile(watch_json_file), f\"watch.json should exist at {watch_json_file}\"\n\n            with open(watch_json_file, 'r', encoding='utf-8') as f:\n                json_obj['watching'][uuid] = json.load(f)\n    else:\n        # Legacy format: url-watches.json\n        logger.info(\"Testing with legacy format (url-watches.json)\")\n        with open(url_watches_json, 'r', encoding='utf-8') as f:\n            json_obj = json.load(f)\n\n    # assert the right amount of watches was found in the JSON\n    assert len(json_obj['watching']) == len(workers), \"Correct number of watches was found in the JSON\"\n\n    i = 0\n    # each one should have a history.txt containing just one line\n    for w in json_obj['watching'].keys():\n        i += 1\n        history_txt_index_file = os.path.join(live_server.app.config['DATASTORE'].datastore_path, w, 'history.txt')\n        assert os.path.isfile(history_txt_index_file), f\"History.txt should exist where I expect it at {history_txt_index_file}\"\n\n        # Should be no errors (could be from brotli etc)\n        assert not live_server.app.config['DATASTORE'].data['watching'][w].get('last_error')\n\n        # Same like in model.Watch\n        with open(history_txt_index_file, \"r\") as f:\n            tmp_history = dict(i.strip().split(',', 2) for i in f.readlines())\n            assert len(tmp_history) == 1, \"History.txt should contain 1 line\"\n\n        # Should be two files,. the history.txt , and the snapshot.txt\n        files_in_watch_dir = os.listdir(os.path.join(live_server.app.config['DATASTORE'].datastore_path, w))\n\n        # Find the snapshot one\n        for fname in files_in_watch_dir:\n            if fname != 'history.txt' and fname != 'watch.json' and fname != 'last-checksum.txt' and 'html' not in fname:\n                if strtobool(os.getenv(\"TEST_WITH_BROTLI\")):\n                    assert fname.endswith('.br'), \"Forced TEST_WITH_BROTLI then it should be a .br filename\"\n\n                full_snapshot_history_path = os.path.join(live_server.app.config['DATASTORE'].datastore_path, w, fname)\n                # contents should match what we requested as content returned from the test url\n                if fname.endswith('.br'):\n                    with open(full_snapshot_history_path, 'rb') as f:\n                        contents = brotli.decompress(f.read()).decode('utf-8')\n                else:\n                    with open(full_snapshot_history_path, 'r') as snapshot_f:\n                        contents = snapshot_f.read()\n\n                watch_title = json_obj['watching'][w]['title']\n                assert json_obj['watching'][w]['title'], \"Watch should have a title set\"\n                assert contents.startswith(watch_title + \"x\"), f\"Snapshot contents in file {fname} should start with '{watch_title}x', got '{contents}'\"\n\n        # With new format, we have watch.json, so 4 files minimum\n        # Note: last-checksum.txt may or may not exist - it gets cleared by settings changes,\n        # and this test changes settings before checking files\n        # This assertion should be AFTER the loop, not inside it\n        if os.path.exists(changedetection_json):\n            # 4 required files: watch.json, html.br, history.txt, extracted text snapshot\n            # last-checksum.txt is optional (cleared by settings changes in this test)\n            assert len(files_in_watch_dir) >= 4 and len(files_in_watch_dir) <= 5, f\"Should be 4-5 files in the dir with new format (last-checksum.txt is optional). Found {len(files_in_watch_dir)}: {files_in_watch_dir}\"\n        else:\n            # 3 required files: html.br, history.txt, extracted text snapshot\n            # last-checksum.txt is optional\n            assert len(files_in_watch_dir) >= 3 and len(files_in_watch_dir) <= 4, f\"Should be 3-4 files in the dir with legacy format (last-checksum.txt is optional). Found {len(files_in_watch_dir)}: {files_in_watch_dir}\"\n\n    # Check that 'default' Watch vars aren't accidentally being saved\n    if os.path.exists(changedetection_json):\n        # New format: check all individual watch.json files\n        for uuid in json_obj['watching'].keys():\n            watch_json_file = os.path.join(datastore_path, uuid, 'watch.json')\n            with open(watch_json_file, 'r', encoding='utf-8') as f:\n                assert '\"default\"' not in f.read(), f\"'default' probably shouldnt be here in {watch_json_file}, it came from when the 'default' Watch vars were accidently being saved\"\n    else:\n        # Legacy format: check url-watches.json\n        with open(url_watches_json, 'r', encoding='utf-8') as f:\n            assert '\"default\"' not in f.read(), \"'default' probably shouldnt be here, it came from when the 'default' Watch vars were accidently being saved\"\n\n\n    delete_all_watches(client)\n\ndef test_check_text_history_view(client, live_server, measure_memory_usage, datastore_path):\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(\"test-one\")\n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    # Give the thread time to pick it up\n    wait_for_all_checks(client)\n\n    # Set second version, Make a change\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(\"test-two\")\n\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    res = client.get(url_for(\"ui.ui_diff.diff_history_page\", uuid=uuid))\n    assert b'test-one' in res.data\n    assert b'test-two' in res.data\n\n    # Set third version, Make a change\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(\"test-three\")\n\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    # It should remember the last viewed time, so the first difference is not shown\n    res = client.get(url_for(\"ui.ui_diff.diff_history_page\", uuid=\"first\"))\n    assert b'test-three' in res.data\n    assert b'test-two' in res.data\n    assert b'test-one' not in res.data\n\n    delete_all_watches(client)\n\n\ndef test_history_trim_global_only(client, live_server, measure_memory_usage, datastore_path):\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = None\n    limit = 3\n\n    for i in range(0, 10):\n        with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n            f.write(f\"test {i}\")\n        if not uuid:\n            uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n        client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n        wait_for_all_checks(client)\n\n        if i ==8:\n            watch = live_server.app.config['DATASTORE'].data['watching'][uuid]\n            history_n = len(list(watch.history.keys()))\n            logger.debug(f\"History length should be at limit {limit} and it is {history_n}\")\n            assert history_n == limit\n\n        if i == 6:\n            res = client.post(\n                url_for(\"settings.settings_page\"),\n                data={\"application-history_snapshot_max_length\": limit},\n                follow_redirects=True\n            )\n            # It will need to detect one more change to start trimming it, which is really at 'start of 7'\n            assert b'Settings updated' in res.data\n\n    delete_all_watches(client)\n\n\ndef test_history_trim_global_override_in_watch(client, live_server, measure_memory_usage, datastore_path):\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = None\n    limit = 3\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\"application-history_snapshot_max_length\": 10000},\n        follow_redirects=True\n    )\n    # It will need to detect one more change to start trimming it, which is really at 'start of 7'\n    assert b'Settings updated' in res.data\n\n\n    for i in range(0, 10):\n        with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n            f.write(f\"test {i}\")\n        if not uuid:\n            uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n            res = client.post(\n                url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n                data={\"include_filters\": \"\", \"url\": test_url, \"tags\": \"\", \"headers\": \"\", 'fetch_backend': \"html_requests\",\n                      \"time_between_check_use_default\": \"y\", \"history_snapshot_max_length\": str(limit)},\n                follow_redirects=True\n            )\n            assert b\"Updated watch.\" in res.data\n\n            wait_for_all_checks(client)\n\n        client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n        wait_for_all_checks(client)\n\n        if i == 8:\n            watch = live_server.app.config['DATASTORE'].data['watching'][uuid]\n            history_n = len(list(watch.history.keys()))\n            logger.debug(f\"History length should be at limit {limit} and it is {history_n}\")\n            assert history_n == limit\n\n        if i == 6:\n            res = client.post(\n                url_for(\"settings.settings_page\"),\n                data={\"application-history_snapshot_max_length\": limit},\n                follow_redirects=True\n            )\n            # It will need to detect one more change to start trimming it, which is really at 'start of 7'\n            assert b'Settings updated' in res.data\n\n    delete_all_watches(client)\n\n"
  },
  {
    "path": "changedetectionio/tests/test_html_to_text.py",
    "content": "#!/usr/bin/env python3\n\"\"\"Test suite for the method to extract text from an html string\"\"\"\nfrom ..html_tools import html_to_text\n\n\ndef test_html_to_text_func():\n    test_html = \"\"\"\n       \n     Some initial text
    \n     
    Which is across multiple lines
    \n      More Text \n     
    \n     So let's see what happens.  
    \n      Even More Text \n     \n     \n    \"\"\"\n\n    # extract text, with 'render_anchor_tag_content' set to False\n    text_content = html_to_text(test_html, render_anchor_tag_content=False)\n\n    no_links_text = \\\n        \"Some initial text\\n\\nWhich is across multiple \" \\\n        \"lines\\n\\nMore Text\\nSo let's see what happens.\\nEven More Text\"\n\n    # check that no links are in the extracted text\n    assert text_content == no_links_text\n\n    # extract text, with 'render_anchor_tag_content' set to True\n    text_content = html_to_text(test_html, render_anchor_tag_content=True)\n\n    links_text = \\\n        \"Some initial text\\n\\nWhich is across multiple lines\\n\\n[ More Text \" \\\n        \"](/first_link)\\nSo let's see what happens.\\n[ Even More Text ]\" \\\n        \"(second_link.com)\"\n\n    # check that links are present in the extracted text\n    assert text_content == links_text\n"
  },
  {
    "path": "changedetectionio/tests/test_i18n.py",
    "content": "#!/usr/bin/env python3\n\nfrom flask import url_for\nfrom .util import live_server_setup, wait_for_all_checks\n\n\ndef test_zh_TW(client, live_server, measure_memory_usage, datastore_path):\n    import time\n    test_url = url_for('test_endpoint', _external=True)\n\n    # Be sure we got a session cookie\n    res = client.get(url_for(\"watchlist.index\"), follow_redirects=True)\n\n    res = client.get(\n        url_for(\"set_language\", locale=\"zh_Hant_TW\"), # Traditional\n        follow_redirects=True\n    )\n    # HTML follows BCP 47 language tag rules, not underscore-based locale formats.\n    assert b'zh\n    res = client.get(\n        url_for(\"set_language\", locale=\"zh\"), # Simplified chinese\n        follow_redirects=True\n    )\n    res = client.get(url_for(\"watchlist.index\"), follow_redirects=True)\n    assert \"选择语言\".encode() in res.data, \"Simplified chinese worked and it means the flask-babel cache worked\"\n\n\n# timeago library just hasn't been updated to use the more modern locale naming convention, before BCP 47 / RFC 5646.\n# The Python timeago library (https://github.com/hustcc/timeago) supports 48 locales but uses different naming conventions than Flask-Babel.\ndef test_zh_Hant_TW_timeago_integration():\n    \"\"\"Test that zh_Hant_TW mapping works and timeago renders Traditional Chinese correctly\"\"\"\n    import timeago\n    from datetime import datetime, timedelta\n    from changedetectionio.languages import get_timeago_locale\n\n    # 1. Test the mapping\n    mapped_locale = get_timeago_locale('zh_Hant_TW')\n    assert mapped_locale == 'zh_TW', \"zh_Hant_TW should map to timeago's zh_TW\"\n    assert get_timeago_locale('zh_TW') == 'zh_TW', \"zh_TW should also map to zh_TW\"\n\n    # 2. Test timeago library renders Traditional Chinese with the mapped locale\n    now = datetime.now()\n\n    # Test various time periods with Traditional Chinese strings\n    result_15s = timeago.format(now - timedelta(seconds=15), now, mapped_locale)\n    assert '秒前' in result_15s, f\"Expected '秒前' in '{result_15s}'\"\n\n    result_5m = timeago.format(now - timedelta(minutes=5), now, mapped_locale)\n    assert '分鐘前' in result_5m, f\"Expected '分鐘前' in '{result_5m}'\"\n\n    result_2h = timeago.format(now - timedelta(hours=2), now, mapped_locale)\n    assert '小時前' in result_2h, f\"Expected '小時前' in '{result_2h}'\"\n\n    result_3d = timeago.format(now - timedelta(days=3), now, mapped_locale)\n    assert '天前' in result_3d, f\"Expected '天前' in '{result_3d}'\"\n\n\ndef test_language_switching(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test that the language switching functionality works correctly.\n\n    1. Switch to Italian using the /set-language endpoint\n    2. Verify that Italian text appears on the page\n    3. Switch back to English and verify English text appears\n    \"\"\"\n\n    # Establish session cookie\n    client.get(url_for(\"watchlist.index\"), follow_redirects=True)\n\n    # Step 1: Set the language to Italian using the /set-language endpoint\n    res = client.get(\n        url_for(\"set_language\", locale=\"it\"),\n        follow_redirects=True\n    )\n\n    assert res.status_code == 200\n\n    # Step 2: Request the index page - should be in Italian\n    # The session cookie should be maintained by the test client\n    res = client.get(\n        url_for(\"watchlist.index\"),\n        follow_redirects=True\n    )\n\n    assert res.status_code == 200\n\n    # Check for Italian text - \"Annulla\" (translation of \"Cancel\")\n    assert b\"Annulla\" in res.data, \"Expected Italian text 'Annulla' not found after setting language to Italian\"\n\n    assert b'Modifiche testo/HTML, JSON e PDF' in res.data, \"Expected italian from processors.available_processors()\"\n\n    # Step 3: Switch back to English\n    res = client.get(\n        url_for(\"set_language\", locale=\"en\"),\n        follow_redirects=True\n    )\n\n    assert res.status_code == 200\n\n    # Request the index page - should now be in English\n    res = client.get(\n        url_for(\"watchlist.index\"),\n        follow_redirects=True\n    )\n\n    assert res.status_code == 200\n\n    # Check for English text\n    assert b\"Cancel\" in res.data, \"Expected English text 'Cancel' not found after switching back to English\"\n\n\ndef test_invalid_locale(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test that setting an invalid locale doesn't break the application.\n    The app should ignore invalid locales and continue working.\n    \"\"\"\n\n    # Establish session cookie\n    client.get(url_for(\"watchlist.index\"), follow_redirects=True)\n\n    # First set to English\n    res = client.get(\n        url_for(\"set_language\", locale=\"en\"),\n        follow_redirects=True\n    )\n\n    assert res.status_code == 200\n\n    # Try to set an invalid locale\n    res = client.get(\n        url_for(\"set_language\", locale=\"invalid_locale_xyz\"),\n        follow_redirects=True\n    )\n\n    assert res.status_code == 200\n\n    # Should still be able to access the page (should stay in English since invalid locale was ignored)\n    res = client.get(\n        url_for(\"watchlist.index\"),\n        follow_redirects=True\n    )\n\n    assert res.status_code == 200\n    assert b\"Cancel\" in res.data, \"Should remain in English when invalid locale is provided\"\n\n\ndef test_language_persistence_in_session(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test that the language preference persists across multiple requests\n    within the same session, and that auto-detect properly clears the preference.\n    \"\"\"\n\n    # Establish session cookie\n    client.get(url_for(\"watchlist.index\"), follow_redirects=True)\n\n    # Set language to Italian\n    res = client.get(\n        url_for(\"set_language\", locale=\"it\"),\n        follow_redirects=True\n    )\n\n    assert res.status_code == 200\n\n    # Make multiple requests - language should persist\n    for _ in range(3):\n        res = client.get(\n            url_for(\"watchlist.index\"),\n            follow_redirects=True\n        )\n\n        assert res.status_code == 200\n        assert b\"Annulla\" in res.data, \"Italian text should persist across requests\"\n\n    # Verify locale is in session\n    with client.session_transaction() as sess:\n        assert sess.get('locale') == 'it', \"Locale should be set in session\"\n\n    # Call auto-detect to clear the locale\n    res = client.get(\n        url_for(\"ui.delete_locale_language_session_var_if_it_exists\"),\n        follow_redirects=True\n    )\n\n    assert res.status_code == 200\n    # Verify the flash message appears (in English since we cleared the locale)\n    assert b\"Language set to auto-detect from browser\" in res.data, \"Should show flash message\"\n\n    # Verify locale was removed from session\n    with client.session_transaction() as sess:\n        assert 'locale' not in sess, \"Locale should be removed from session after auto-detect\"\n\n    # Now requests should use browser default (English in test environment)\n    res = client.get(\n        url_for(\"watchlist.index\"),\n        follow_redirects=True\n    )\n\n    assert res.status_code == 200\n    assert b\"Cancel\" in res.data, \"Should show English after auto-detect clears Italian\"\n    assert b\"Annulla\" not in res.data, \"Should not show Italian after auto-detect\"\n\n\ndef test_set_language_with_redirect(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test that changing language keeps the user on the same page.\n    Example: User is on /settings, changes language, stays on /settings.\n    \"\"\"\n    from flask import url_for\n\n    # Establish session cookie\n    client.get(url_for(\"watchlist.index\"), follow_redirects=True)\n\n    # Set language with a redirect parameter (simulating language change from /settings)\n    res = client.get(\n        url_for(\"set_language\", locale=\"de\", redirect=\"/settings\"),\n        follow_redirects=False\n    )\n\n    # Should redirect back to settings\n    assert res.status_code in [302, 303]\n    assert '/settings' in res.location\n\n    # Verify language was set in session\n    with client.session_transaction() as sess:\n        assert sess.get('locale') == 'de'\n\n    # Test with invalid locale (should still redirect safely)\n    res = client.get(\n        url_for(\"set_language\", locale=\"invalid_locale\", redirect=\"/settings\"),\n        follow_redirects=False\n    )\n    assert res.status_code in [302, 303]\n    assert '/settings' in res.location\n\n    # Test with malicious redirect (should default to watchlist)\n    res = client.get(\n        url_for(\"set_language\", locale=\"en\", redirect=\"https://evil.com\"),\n        follow_redirects=False\n    )\n    assert res.status_code in [302, 303]\n    # Should not redirect to evil.com\n    assert 'evil.com' not in res.location\n\n\ndef test_time_unit_translations(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test that time unit labels (Hours, Minutes, Seconds) and Chrome Extension\n    are correctly translated on the settings page for all supported languages.\n    \"\"\"\n    from flask import url_for\n\n    # Establish session cookie\n    client.get(url_for(\"watchlist.index\"), follow_redirects=True)\n\n    # Test Italian translations\n    res = client.get(url_for(\"set_language\", locale=\"it\"), follow_redirects=True)\n    assert res.status_code == 200\n\n    res = client.get(url_for(\"settings.settings_page\"), follow_redirects=True)\n    assert res.status_code == 200\n\n    # Check that Italian translations are present (not English)\n    assert b\"Minutes\" not in res.data or b\"Minuti\" in res.data, \"Expected Italian 'Minuti' not English 'Minutes'\"\n    assert b\"Ore\" in res.data, \"Expected Italian 'Ore' for Hours\"\n    assert b\"Minuti\" in res.data, \"Expected Italian 'Minuti' for Minutes\"\n    assert b\"Secondi\" in res.data, \"Expected Italian 'Secondi' for Seconds\"\n    assert b\"Estensione Chrome\" in res.data, \"Expected Italian 'Estensione Chrome' for Chrome Extension\"\n    assert b\"Intervallo tra controlli\" in res.data, \"Expected Italian 'Intervallo tra controlli' for Time Between Check\"\n    assert b\"Time Between Check\" not in res.data, \"Should not have English 'Time Between Check'\"\n\n    # Test Korean translations\n    res = client.get(url_for(\"set_language\", locale=\"ko\"), follow_redirects=True)\n    assert res.status_code == 200\n\n    res = client.get(url_for(\"settings.settings_page\"), follow_redirects=True)\n    assert res.status_code == 200\n\n    # Check that Korean translations are present (not English)\n    # Korean: Hours=시간, Minutes=분, Seconds=초, Chrome Extension=Chrome 확장 프로그램, Time Between Check=확인 간격\n    assert \"시간\".encode() in res.data, \"Expected Korean '시간' for Hours\"\n    assert \"분\".encode() in res.data, \"Expected Korean '분' for Minutes\"\n    assert \"초\".encode() in res.data, \"Expected Korean '초' for Seconds\"\n    assert \"Chrome 확장 프로그램\".encode() in res.data, \"Expected Korean 'Chrome 확장 프로그램' for Chrome Extension\"\n    assert \"확인 간격\".encode() in res.data, \"Expected Korean '확인 간격' for Time Between Check\"\n    # Make sure we don't have the incorrect translations\n    assert \"목요일\".encode() not in res.data, \"Should not have '목요일' (Thursday) for Hours\"\n    assert \"무음\".encode() not in res.data, \"Should not have '무음' (Mute) for Minutes\"\n    assert \"Chrome 요청\".encode() not in res.data, \"Should not have 'Chrome 요청' (Chrome requests) for Chrome Extension\"\n    assert b\"Time Between Check\" not in res.data, \"Should not have English 'Time Between Check'\"\n\n    # Test Chinese Simplified translations\n    res = client.get(url_for(\"set_language\", locale=\"zh\"), follow_redirects=True)\n    assert res.status_code == 200\n\n    res = client.get(url_for(\"settings.settings_page\"), follow_redirects=True)\n    assert res.status_code == 200\n\n    # Check that Chinese translations are present\n    # Chinese: Hours=小时, Minutes=分钟, Seconds=秒, Chrome Extension=Chrome 扩展程序, Time Between Check=检查间隔\n    assert \"小时\".encode() in res.data, \"Expected Chinese '小时' for Hours\"\n    assert \"分钟\".encode() in res.data, \"Expected Chinese '分钟' for Minutes\"\n    assert \"秒\".encode() in res.data, \"Expected Chinese '秒' for Seconds\"\n    assert \"Chrome 扩展程序\".encode() in res.data, \"Expected Chinese 'Chrome 扩展程序' for Chrome Extension\"\n    assert \"检查间隔\".encode() in res.data, \"Expected Chinese '检查间隔' for Time Between Check\"\n    assert b\"Time Between Check\" not in res.data, \"Should not have English 'Time Between Check'\"\n\n    # Test German translations\n    res = client.get(url_for(\"set_language\", locale=\"de\"), follow_redirects=True)\n    assert res.status_code == 200\n\n    res = client.get(url_for(\"settings.settings_page\"), follow_redirects=True)\n    assert res.status_code == 200\n\n    # Check that German translations are present\n    # German: Hours=Stunden, Minutes=Minuten, Seconds=Sekunden, Chrome Extension=Chrome-Erweiterung, Time Between Check=Prüfintervall\n    assert b\"Stunden\" in res.data, \"Expected German 'Stunden' for Hours\"\n    assert b\"Minuten\" in res.data, \"Expected German 'Minuten' for Minutes\"\n    assert b\"Sekunden\" in res.data, \"Expected German 'Sekunden' for Seconds\"\n    assert b\"Chrome-Erweiterung\" in res.data, \"Expected German 'Chrome-Erweiterung' for Chrome Extension\"\n    assert b\"Time Between Check\" not in res.data, \"Should not have English 'Time Between Check'\"\n\n    # Test Traditional Chinese (zh_Hant_TW) translations\n    res = client.get(url_for(\"set_language\", locale=\"zh_Hant_TW\"), follow_redirects=True)\n    assert res.status_code == 200\n\n    res = client.get(url_for(\"settings.settings_page\"), follow_redirects=True)\n    assert res.status_code == 200\n\n    # Check that Traditional Chinese translations are present (not English)\n    # Traditional Chinese: Hours=小時, Minutes=分鐘, Seconds=秒, Chrome Extension=Chrome 擴充功能, Time Between Check=檢查間隔\n    assert \"小時\".encode() in res.data, \"Expected Traditional Chinese '小時' for Hours\"\n    assert \"分鐘\".encode() in res.data, \"Expected Traditional Chinese '分鐘' for Minutes\"\n    assert \"秒\".encode() in res.data, \"Expected Traditional Chinese '秒' for Seconds\"\n    assert \"Chrome 擴充功能\".encode() in res.data, \"Expected Traditional Chinese 'Chrome 擴充功能' for Chrome Extension\"\n    assert \"發送測試通知\".encode() in res.data, \"Expected Traditional Chinese '發送測試通知' for Send test notification\"\n    assert \"通知除錯記錄\".encode() in res.data, \"Expected Traditional Chinese '通知除錯記錄' for Notification debug logs\"\n    assert \"檢查間隔\".encode() in res.data, \"Expected Traditional Chinese '檢查間隔' for Time Between Check\"\n    # Make sure we don't have incorrect English text or wrong translations\n    assert b\"Send test notification\" not in res.data, \"Should not have English 'Send test notification'\"\n    assert b\"Time Between Check\" not in res.data, \"Should not have English 'Time Between Check'\"\n    assert \"Chrome 請求\".encode() not in res.data, \"Should not have incorrect 'Chrome 請求' (Chrome requests)\"\n    assert \"使用預設通知\".encode() not in res.data, \"Should not have incorrect '使用預設通知' (Use default notification)\"\n\n\ndef test_accept_language_header_zh_tw(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test that browsers sending zh-TW in Accept-Language header get Traditional Chinese.\n    This tests the locale alias mapping for issue #3779.\n    \"\"\"\n    from flask import url_for\n\n    # Clear any session data to simulate a fresh visitor\n    with client.session_transaction() as sess:\n        sess.clear()\n\n    # Request the index page with zh-TW in Accept-Language header (what browsers send)\n    res = client.get(\n        url_for(\"watchlist.index\"),\n        headers={'Accept-Language': 'zh-TW,zh;q=0.9,en;q=0.8'},\n        follow_redirects=True\n    )\n\n    assert res.status_code == 200\n\n    # Should get Traditional Chinese content, not Simplified Chinese\n    # Traditional: 選擇語言, Simplified: 选择语言\n    assert '選擇語言'.encode() in res.data, \"Expected Traditional Chinese '選擇語言' (Select Language)\"\n    assert '选择语言'.encode() not in res.data, \"Should not get Simplified Chinese '选择语言'\"\n\n    # Check HTML lang attribute uses BCP 47 format\n    assert b'' in res.data, \\\n        \"Expected Taiwan flag 'fi fi-tw' for Traditional Chinese\"\n    assert b'' not in res.data, \\\n        \"Should not show China flag 'fi fi-cn' for Traditional Chinese\"\n\n    # Verify we're getting Traditional Chinese text throughout the page\n    res = client.get(\n        url_for(\"settings.settings_page\"),\n        headers={'Accept-Language': 'zh-TW,zh;q=0.9,en;q=0.8'},\n        follow_redirects=True\n    )\n\n    assert res.status_code == 200\n\n    # Check Traditional Chinese translations (not English)\n    assert \"小時\".encode() in res.data, \"Expected Traditional Chinese '小時' for Hours\"\n    assert \"分鐘\".encode() in res.data, \"Expected Traditional Chinese '分鐘' for Minutes\"\n    assert b\"Hours\" not in res.data or \"小時\".encode() in res.data, \"Should have Traditional Chinese, not English\"\n\n\ndef test_accept_language_header_en_variants(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test that browsers sending en-GB and en-US in Accept-Language header get the correct English variant.\n    This ensures the locale selector works properly for English variants.\n    \"\"\"\n    from flask import url_for\n\n    # Test 1: British English (en-GB)\n    with client.session_transaction() as sess:\n        sess.clear()\n\n    res = client.get(\n        url_for(\"watchlist.index\"),\n        headers={'Accept-Language': 'en-GB,en;q=0.9'},\n        follow_redirects=True\n    )\n\n    assert res.status_code == 200\n\n    # Should get English content\n    assert b\"Select Language\" in res.data, \"Expected English text 'Select Language'\"\n\n    # Check HTML lang attribute uses BCP 47 format with hyphen\n    assert b'' in res.data, \\\n        \"Expected UK flag 'fi fi-gb' for British English\"\n\n    # Test 2: American English (en-US)\n    with client.session_transaction() as sess:\n        sess.clear()\n\n    res = client.get(\n        url_for(\"watchlist.index\"),\n        headers={'Accept-Language': 'en-US,en;q=0.9'},\n        follow_redirects=True\n    )\n\n    assert res.status_code == 200\n\n    # Should get English content\n    assert b\"Select Language\" in res.data, \"Expected English text 'Select Language'\"\n\n    # Check HTML lang attribute uses BCP 47 format with hyphen\n    assert b'' in res.data, \\\n        \"Expected US flag 'fi fi-us' for American English\"\n\n    # Test 3: Generic 'en' should fall back to one of the English variants\n    with client.session_transaction() as sess:\n        sess.clear()\n\n    res = client.get(\n        url_for(\"watchlist.index\"),\n        headers={'Accept-Language': 'en'},\n        follow_redirects=True\n    )\n\n    assert res.status_code == 200\n\n    # Should get English content (either variant is fine)\n    assert b\"Select Language\" in res.data, \"Expected English text 'Select Language'\"\n\n\ndef test_accept_language_header_zh_simplified(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test that browsers sending zh or zh-CN in Accept-Language header get Simplified Chinese.\n    This ensures Simplified Chinese still works correctly and doesn't get confused with Traditional.\n    \"\"\"\n    from flask import url_for\n\n    # Test 1: Generic 'zh' should get Simplified Chinese\n    with client.session_transaction() as sess:\n        sess.clear()\n\n    res = client.get(\n        url_for(\"watchlist.index\"),\n        headers={'Accept-Language': 'zh,en;q=0.9'},\n        follow_redirects=True\n    )\n\n    assert res.status_code == 200\n\n    # Should get Simplified Chinese content, not Traditional Chinese\n    # Simplified: 选择语言, Traditional: 選擇語言\n    assert '选择语言'.encode() in res.data, \"Expected Simplified Chinese '选择语言' (Select Language)\"\n    assert '選擇語言'.encode() not in res.data, \"Should not get Traditional Chinese '選擇語言'\"\n\n    # Check HTML lang attribute\n    assert b'' in res.data, \\\n        \"Expected China flag 'fi fi-cn' for Simplified Chinese\"\n    assert b'' not in res.data, \\\n        \"Should not show Taiwan flag 'fi fi-tw' for Simplified Chinese\"\n\n    # Test 2: 'zh-CN' should also get Simplified Chinese\n    with client.session_transaction() as sess:\n        sess.clear()\n\n    res = client.get(\n        url_for(\"watchlist.index\"),\n        headers={'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8'},\n        follow_redirects=True\n    )\n\n    assert res.status_code == 200\n\n    # Should get Simplified Chinese content\n    assert '选择语言'.encode() in res.data, \"Expected Simplified Chinese '选择语言' with zh-CN header\"\n    assert '選擇語言'.encode() not in res.data, \"Should not get Traditional Chinese with zh-CN header\"\n\n    # Check that the correct flag icon is shown (China flag for zh-CN)\n    assert b'' in res.data, \\\n        \"Expected China flag 'fi fi-cn' for zh-CN header\"\n\n    # Verify Simplified Chinese in settings page\n    res = client.get(\n        url_for(\"settings.settings_page\"),\n        headers={'Accept-Language': 'zh,en;q=0.9'},\n        follow_redirects=True\n    )\n\n    assert res.status_code == 200\n\n    # Check Simplified Chinese translations (not Traditional or English)\n    # Simplified: 小时, Traditional: 小時\n    assert \"小时\".encode() in res.data, \"Expected Simplified Chinese '小时' for Hours\"\n    assert \"分钟\".encode() in res.data, \"Expected Simplified Chinese '分钟' for Minutes\"\n    assert \"秒\".encode() in res.data, \"Expected Simplified Chinese '秒' for Seconds\"\n    # Make sure it's not Traditional Chinese\n    assert \"小時\".encode() not in res.data, \"Should not have Traditional Chinese '小時'\"\n    assert \"分鐘\".encode() not in res.data, \"Should not have Traditional Chinese '分鐘'\"\n\n\ndef test_session_locale_overrides_accept_language(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test that session locale preference overrides browser Accept-Language header.\n\n    Scenario:\n    1. Browser auto-detects zh-TW (Traditional Chinese) from Accept-Language header\n    2. User explicitly selects Korean language\n    3. On subsequent page loads, Korean should be shown (not Traditional Chinese)\n       even though the Accept-Language header still says zh-TW\n\n    This tests the session override behavior for issue #3779.\n    \"\"\"\n    from flask import url_for\n\n    # Step 1: Clear session and make first request with zh-TW header (auto-detect)\n    with client.session_transaction() as sess:\n        sess.clear()\n\n    res = client.get(\n        url_for(\"watchlist.index\"),\n        headers={'Accept-Language': 'zh-TW,zh;q=0.9,en;q=0.8'},\n        follow_redirects=True\n    )\n\n    assert res.status_code == 200\n\n    # Should initially get Traditional Chinese from auto-detect\n    assert '選擇語言'.encode() in res.data, \"Expected Traditional Chinese '選擇語言' from auto-detect\"\n    assert b'' in res.data, \\\n        \"Expected Taiwan flag 'fi fi-tw' from auto-detect\"\n\n    # Step 2: User explicitly selects Korean language\n    res = client.get(\n        url_for(\"set_language\", locale=\"ko\"),\n        headers={'Accept-Language': 'zh-TW,zh;q=0.9,en;q=0.8'},  # Browser still sends zh-TW\n        follow_redirects=True\n    )\n\n    assert res.status_code == 200\n\n    # Step 3: Make another request with same zh-TW header\n    # Session should override the Accept-Language header\n    res = client.get(\n        url_for(\"watchlist.index\"),\n        headers={'Accept-Language': 'zh-TW,zh;q=0.9,en;q=0.8'},  # Still sending zh-TW!\n        follow_redirects=True\n    )\n\n    assert res.status_code == 200\n\n    # Should now get Korean (session overrides auto-detect)\n    # Korean: 언어 선택, Traditional Chinese: 選擇語言\n    assert '언어 선택'.encode() in res.data, \"Expected Korean '언어 선택' (Select Language) from session\"\n    assert '選擇語言'.encode() not in res.data, \"Should not get Traditional Chinese when Korean is set in session\"\n\n    # Check HTML lang attribute is Korean\n    assert b'' in res.data, \\\n        \"Expected Korean flag 'fi fi-kr' from session preference\"\n    assert b'' not in res.data, \\\n        \"Should not show Taiwan flag when Korean is set in session\"\n\n    # Verify Korean text on settings page as well\n    res = client.get(\n        url_for(\"settings.settings_page\"),\n        headers={'Accept-Language': 'zh-TW,zh;q=0.9,en;q=0.8'},  # Still zh-TW!\n        follow_redirects=True\n    )\n\n    assert res.status_code == 200\n\n    # Check Korean translations (not Traditional Chinese or English)\n    # Korean: 시간 (Hours), 분 (Minutes), 초 (Seconds)\n    # Traditional Chinese: 小時, 分鐘, 秒\n    assert \"시간\".encode() in res.data, \"Expected Korean '시간' for Hours\"\n    assert \"분\".encode() in res.data, \"Expected Korean '분' for Minutes\"\n    assert \"小時\".encode() not in res.data, \"Should not have Traditional Chinese '小時' when Korean is set\"\n    assert \"分鐘\".encode() not in res.data, \"Should not have Traditional Chinese '分鐘' when Korean is set\"\n\n\ndef test_clear_history_translated_confirmation(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test that clearing snapshot history works with translated confirmation text.\n\n    Issue #3865: When the app language is set to German, the clear history\n    confirmation dialog shows the translated word (e.g. 'loschen') but the\n    backend only accepted the English word 'clear', making it impossible\n    to clear snapshots in non-English languages.\n    \"\"\"\n    from flask import url_for\n\n    test_url = url_for('test_endpoint', _external=True)\n\n    # Add a watch so there is history to clear\n    res = client.post(\n        url_for(\"imports.import_page\"),\n        data={\"urls\": test_url},\n        follow_redirects=True\n    )\n    assert b\"1 Imported\" in res.data\n    wait_for_all_checks(client)\n\n    # Set language to German\n    res = client.get(\n        url_for(\"set_language\", locale=\"de\"),\n        follow_redirects=True\n    )\n    assert res.status_code == 200\n\n    # Verify the clear history page shows the German confirmation word\n    res = client.get(\n        url_for(\"ui.clear_all_history\"),\n        follow_redirects=True\n    )\n    assert res.status_code == 200\n    assert \"löschen\".encode() in res.data, \"Expected German word 'loschen' on clear history page\"\n\n    # Submit the form with the German translated word\n    res = client.post(\n        url_for(\"ui.clear_all_history\"),\n        data={\"confirmtext\": \"löschen\"},\n        follow_redirects=True\n    )\n    assert res.status_code == 200\n    # Should NOT show error message\n    assert b\"Incorrect confirmation text\" not in res.data, \\\n        \"German confirmation word 'loschen' should be accepted (issue #3865)\"\n\n    # Switch back to English and verify English word still works\n    res = client.get(\n        url_for(\"set_language\", locale=\"en_US\"),\n        follow_redirects=True\n    )\n\n    res = client.post(\n        url_for(\"ui.clear_all_history\"),\n        data={\"confirmtext\": \"clear\"},\n        follow_redirects=True\n    )\n    assert res.status_code == 200\n    assert b\"Incorrect confirmation text\" not in res.data, \\\n        \"English confirmation word 'clear' should still be accepted\"\n\n    # Verify that missing/empty confirmtext does not crash the server\n    res = client.post(\n        url_for(\"ui.clear_all_history\"),\n        data={},\n        follow_redirects=True\n    )\n    assert res.status_code == 200, \\\n        \"Missing confirmtext should not crash the server\"\n"
  },
  {
    "path": "changedetectionio/tests/test_ignore.py",
    "content": "#!/usr/bin/env python3\n\nimport time\nfrom flask import url_for\nfrom .util import live_server_setup, wait_for_all_checks\nfrom changedetectionio import html_tools\nfrom . util import  extract_UUID_from_client\nimport os\n\ndef set_original_ignore_response(datastore_path):\n    test_return_data = \"\"\"\n       \n     Some initial text
    \n     
    Which is across multiple lines
    \n     
    \n     So let's see what happens.  
    \n     
    oh yeah 456
    \n     \n     \n\n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n\n\ndef test_ignore(client, live_server, measure_memory_usage, datastore_path):\n   #  live_server_setup(live_server) # Setup on conftest per function\n    set_original_ignore_response(datastore_path)\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    # Give the thread time to pick it up\n    wait_for_all_checks(client)\n    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))\n    # use the highlighter endpoint\n    res = client.post(\n        url_for(\"ui.ui_edit.highlight_submit_ignore_url\", uuid=uuid),\n        data={\"mode\": 'digit-regex', 'selection': 'oh yeah 123'},\n        follow_redirects=True\n    )\n\n    res = client.get(url_for(\"ui.ui_edit.edit_page\", uuid=uuid))\n    # should be a regex now\n    assert b'/oh\\ yeah\\ \\d+/' in res.data\n\n    # Should return a link\n    assert b'href' in res.data\n\n    # It should not be in the preview anymore\n    res = client.get(url_for(\"ui.ui_preview.preview_page\", uuid=uuid))\n    assert b'
    oh yeah 456' not in res.data\n\n    # Should be in base.html\n    assert b'csrftoken' in res.data\n\n\ndef test_strip_ignore_lines(client, live_server, measure_memory_usage, datastore_path):\n   #  live_server_setup(live_server) # Setup on conftest per function\n    set_original_ignore_response(datastore_path)\n\n\n    # Goto the settings page, add our ignore text\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\n            \"requests-time_between_check-minutes\": 180,\n            \"application-ignore_whitespace\": \"y\",\n            \"application-strip_ignored_lines\": \"y\",\n            \"application-global_ignore_text\": \"Which is across multiple\",\n            'application-fetch_backend': \"html_requests\"\n        },\n        follow_redirects=True\n    )\n    assert b\"Settings updated.\" in res.data\n\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    # Give the thread time to pick it up\n    wait_for_all_checks(client)\n    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))\n\n    # It should not be in the preview anymore\n    res = client.get(url_for(\"ui.ui_preview.preview_page\", uuid=uuid))\n    assert b'
    ' not in res.data\n    assert b'Which is across multiple' not in res.data\n"
  },
  {
    "path": "changedetectionio/tests/test_ignore_regex_text.py",
    "content": "#!/usr/bin/env python3\n\nfrom . util import live_server_setup\nfrom changedetectionio import html_tools\n\n\n\n# Unit test of the stripper\n# Always we are dealing in utf-8\ndef test_strip_regex_text_func():\n    test_content = \"\"\"\n    but sometimes we want to remove the lines.\n    \n    but 1 lines\n    skip 5 lines\n    really? yes man\n#/not this tries weirdly formed regex or just strings starting with /\n/not this\n    but including 1234 lines\n    igNORe-cAse text we dont want to keep    \n    but not always.\"\"\"\n\n\n    ignore_lines = [\n        \"sometimes\",\n        \"/\\s\\d{2,3}\\s/\",\n        \"/ignore-case text/\",\n        \"really?\",\n        \"/skip \\d lines/i\",\n        \"/not\"\n    ]\n\n    stripped_content = html_tools.strip_ignore_text(test_content, ignore_lines)\n    assert \"but 1 lines\" in stripped_content\n    assert \"igNORe-cAse text\" not in stripped_content\n    assert \"but 1234 lines\" not in stripped_content\n    assert \"really\" not in stripped_content\n    assert \"not this\" not in stripped_content\n\n    # Check line number reporting\n    stripped_content = html_tools.strip_ignore_text(test_content, ignore_lines, mode=\"line numbers\")\n    assert stripped_content == [2, 5, 6, 7, 8, 10]\n    \n    stripped_content = html_tools.strip_ignore_text(test_content, ['/but 1.+5 lines/s'])\n    assert \"but 1 lines\" not in stripped_content\n    assert \"skip 5 lines\" not in stripped_content\n    \n    stripped_content = html_tools.strip_ignore_text(test_content, ['/but 1.+5 lines/s'], mode=\"line numbers\")\n    assert stripped_content == [4, 5]\n    \n    stripped_content = html_tools.strip_ignore_text(test_content, ['/.+/s'])\n    assert stripped_content == \"\"\n    \n    stripped_content = html_tools.strip_ignore_text(test_content, ['/.+/s'], mode=\"line numbers\")\n    assert stripped_content == [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]\n\n    stripped_content = html_tools.strip_ignore_text(test_content, ['/^.+but.+\\\\n.+lines$/m'])\n    assert \"but 1 lines\" not in stripped_content\n    assert \"skip 5 lines\" not in stripped_content\n\n    stripped_content = html_tools.strip_ignore_text(test_content, ['/^.+but.+\\\\n.+lines$/m'], mode=\"line numbers\")\n    assert stripped_content == [4, 5]\n\n    stripped_content = html_tools.strip_ignore_text(test_content, ['/^.+?\\.$/m'])\n    assert \"but sometimes we want to remove the lines.\" not in stripped_content\n    assert \"but not always.\" not in stripped_content\n\n    stripped_content = html_tools.strip_ignore_text(test_content, ['/^.+?\\.$/m'], mode=\"line numbers\")\n    assert stripped_content == [2, 11]\n\n    stripped_content = html_tools.strip_ignore_text(test_content, ['/but.+?but/ms'])\n    assert \"but sometimes we want to remove the lines.\" not in stripped_content\n    assert \"but 1 lines\" not in stripped_content\n    assert \"but 1234 lines\" not in stripped_content\n    assert \"igNORe-cAse text we dont want to keep\" not in stripped_content\n    assert \"but not always.\" not in stripped_content\n\n    stripped_content = html_tools.strip_ignore_text(test_content, ['/but.+?but/ms'], mode=\"line numbers\")\n    assert stripped_content == [2, 3, 4, 9, 10, 11]\n\n    stripped_content = html_tools.strip_ignore_text(\"\\n\\ntext\\n\\ntext\\n\\n\", ['/^$/ms'], mode=\"line numbers\")\n    assert stripped_content == [1, 2, 4, 6]\n\n    # Check that linefeeds are preserved when there are is no matching ignores\n    content = \"some text\\n\\nand other text\\n\"\n    stripped_content = html_tools.strip_ignore_text(content, ignore_lines)\n    assert content == stripped_content\n"
  },
  {
    "path": "changedetectionio/tests/test_ignore_text.py",
    "content": "#!/usr/bin/env python3\n\nimport time\nfrom flask import url_for\nfrom .util import live_server_setup, wait_for_all_checks, delete_all_watches\nfrom changedetectionio import html_tools\nimport os\n\n\n\n# Unit test of the stripper\n# Always we are dealing in utf-8\ndef test_strip_text_func():\n    test_content = \"\"\"\n    Some content\n    is listed here\n\n    but sometimes we want to remove the lines.\n\n    but not always.\"\"\"\n\n    ignore_lines = [\"sometimes\"]\n\n    stripped_content = html_tools.strip_ignore_text(test_content, ignore_lines)\n    assert \"sometimes\" not in stripped_content\n    assert \"Some content\" in stripped_content\n\n    # Check that line feeds dont get chewed up when something is found\n    test_content = \"Some initial text\\n\\nWhich is across multiple lines\\n\\nZZZZz\\n\\n\\nSo let's see what happens.\"\n    ignore = ['something irrelevent but just to check', 'XXXXX', 'YYYYY', 'ZZZZZ']\n\n    stripped_content = html_tools.strip_ignore_text(test_content, ignore)\n    assert stripped_content == \"Some initial text\\n\\nWhich is across multiple lines\\n\\n\\n\\nSo let's see what happens.\"\n\ndef set_original_ignore_response(datastore_path, ver_stamp=\"123\"):\n    test_return_data = f\"\"\"\n       \n     Some initial text
    \n     
    Which is across multiple lines
    \n     
    \n     So let's see what happens.  
    \n     \n     \n     \n\n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n\n\ndef set_modified_original_ignore_response(datastore_path, ver_stamp=\"123\"):\n    test_return_data = f\"\"\"\n       \n     Some NEW nice initial text
    \n     
    Which is across multiple lines
    \n     
    \n     So let's see what happens.  
    \n     \n     
    new ignore stuff
    \n     
    blah
    \n     \n     \n\n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n\n\n# Is the same but includes ZZZZZ, 'ZZZZZ' is the last line in ignore_text\ndef set_modified_ignore_response(datastore_path, ver_stamp=\"123\"):\n    test_return_data = f\"\"\"\n       \n     Some initial text
    \n     
    Which is across multiple lines
    \n     
    ZZZZz
    \n     
    \n     So let's see what happens.  
    \n     \n     \n     \n\n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n\n\n# Ignore text now just removes it entirely, is a LOT more simpler code this way\n\ndef test_check_ignore_text_functionality(client, live_server, measure_memory_usage, datastore_path):\n\n    # Use a mix of case in ZzZ to prove it works case-insensitive.\n    ignore_text = \"XXXXX\\r\\nYYYYY\\r\\nzZzZZ\\r\\nnew ignore stuff\"\n    set_original_ignore_response(datastore_path=datastore_path)\n\n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    # Give the thread time to pick it up\n    wait_for_all_checks(client)\n\n    # Goto the edit page, add our ignore text\n    # Add our URL to the import page\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={\"ignore_text\": ignore_text, \"url\": test_url, 'fetch_backend': \"html_requests\", \"time_between_check_use_default\": \"y\"},\n        follow_redirects=True\n    )\n    assert b\"Updated watch.\" in res.data\n\n    # Check it saved\n    res = client.get(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n    )\n    assert bytes(ignore_text.encode('utf-8')) in res.data\n\n    # Trigger a check\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    # Give the thread time to pick it up\n    wait_for_all_checks(client)\n\n    # It should report nothing found (no new 'has-unread-changes' class)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'has-unread-changes' not in res.data\n    assert b'/test-endpoint' in res.data\n\n\n    res = client.get(url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"))\n    # nothing ignored because none of the text matched\n    assert b'ignored_line_numbers = []' in res.data\n\n    #  Make a change\n    set_modified_ignore_response(datastore_path=datastore_path)\n\n    # Trigger a check\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    # Give the thread time to pick it up\n    wait_for_all_checks(client)\n\n    # It should report nothing found (no new 'has-unread-changes' class)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'has-unread-changes' not in res.data\n    assert b'/test-endpoint' in res.data\n\n\n\n    # Just to be sure.. set a regular modified change..\n    set_modified_original_ignore_response(datastore_path=datastore_path)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'has-unread-changes' in res.data\n\n    res = client.get(url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"))\n\n    # SHOULD BE be in the preview, it was added in set_modified_original_ignore_response()\n    # and we have \"new ignore stuff\" in ignore_text\n    # it is only ignored, it is not removed (it will be highlighted too)\n    assert b'new ignore stuff' in res.data\n    # Data for the highlighting is present (this is done in JS for now)\n    assert b'ignored_line_numbers = [8]' in res.data\n\n    delete_all_watches(client)\n\n# When adding some ignore text, it should not trigger a change, even if something else on that line changes\ndef _run_test_global_ignore(client, datastore_path, as_source=False, extra_ignore=\"\"):\n    ignore_text = \"XXXXX\\r\\nYYYYY\\r\\nZZZZZ\\r\\n\"+extra_ignore\n\n    set_original_ignore_response(datastore_path=datastore_path)\n\n    # Goto the settings page, add our ignore text\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\n            \"requests-time_between_check-minutes\": 180,\n            \"application-ignore_whitespace\": \"y\",\n            \"application-global_ignore_text\": ignore_text,\n            'application-fetch_backend': \"html_requests\"\n        },\n        follow_redirects=True\n    )\n    assert b\"Settings updated.\" in res.data\n\n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True)\n    if as_source:\n        # Switch to source mode so we can test that too!\n        test_url = \"source:\"+test_url\n\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    # Give the thread time to pick it up\n    wait_for_all_checks(client)\n\n    #Adding some ignore text should not trigger a change\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={\"ignore_text\": \"something irrelevent but just to check\", \"url\": test_url, 'fetch_backend': \"html_requests\", \"time_between_check_use_default\": \"y\"},\n        follow_redirects=True\n    )\n    assert b\"Updated watch.\" in res.data\n    wait_for_all_checks(client)\n    # Check it saved\n    res = client.get(\n        url_for(\"settings.settings_page\"),\n    )\n\n    for i in ignore_text.splitlines():\n        assert bytes(i.encode('utf-8')) in res.data\n\n\n    # Trigger a check\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n    # It should report nothing found (no new 'has-unread-changes' class), adding random ignore text should not cause a change\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'has-unread-changes' not in res.data\n    assert b'/test-endpoint' in res.data\n#####\n\n    # Make a change which includes the ignore text, it should be ignored and no 'change' triggered\n    # It adds text with \"ZZZZzzzz\" and \"ZZZZ\" is in the ignore list\n    # And tweaks the ver_stamp which should be picked up by global regex ignore\n    set_modified_ignore_response(ver_stamp=time.time(), datastore_path=datastore_path)\n\n    # Trigger a check\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    # Give the thread time to pick it up\n    wait_for_all_checks(client)\n\n    # It should report nothing found (no new 'has-unread-changes' class)\n    res = client.get(url_for(\"watchlist.index\"))\n\n    assert b'has-unread-changes' not in res.data\n    assert b'/test-endpoint' in res.data\n\n    # Just to be sure.. set a regular modified change that will trigger it\n    set_modified_original_ignore_response(datastore_path=datastore_path)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'has-unread-changes' in res.data\n\n    delete_all_watches(client)\n\ndef test_check_global_ignore_text_functionality(client, live_server, measure_memory_usage, datastore_path):\n    \n    _run_test_global_ignore(client, as_source=False, datastore_path=datastore_path)\n\ndef test_check_global_ignore_text_functionality_as_source(client, live_server, measure_memory_usage, datastore_path):\n    \n    _run_test_global_ignore(client, as_source=True, extra_ignore='/\\?v=\\d/', datastore_path=datastore_path)\n"
  },
  {
    "path": "changedetectionio/tests/test_ignorehyperlinks.py",
    "content": "#!/usr/bin/env python3\n\"\"\"Test suite for the render/not render anchor tag content functionality\"\"\"\n\nimport time\nfrom flask import url_for\nfrom .util import live_server_setup, wait_for_all_checks, delete_all_watches\nimport os\n\n\ndef set_original_ignore_response(datastore_path):\n    test_return_data = \"\"\"\n       \n     Some initial text
    \n      Some More Text \n     
    \n     So let's see what happens.  
    \n     \n     \n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n\n\n# Should be the same as set_original_ignore_response(datastore_path=datastore_path) but with a different\n# link\ndef set_modified_ignore_response(datastore_path):\n    test_return_data = \"\"\"\n       \n     Some initial text
    \n      Some More Text \n     
    \n     So let's see what happens.  
    \n     \n     \n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n\ndef test_render_anchor_tag_content_true(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"Testing that the link changes are detected when\n    render_anchor_tag_content setting is set to true\"\"\"\n\n    # Give the endpoint time to spin up\n    time.sleep(1)\n\n    # set original html text\n    set_original_ignore_response(datastore_path=datastore_path)\n\n    # Goto the settings page, choose to ignore links (dont select/send \"application-render_anchor_tag_content\")\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\n            \"requests-time_between_check-minutes\": 180,\n            \"application-fetch_backend\": \"html_requests\",\n        },\n        follow_redirects=True,\n    )\n    assert b\"Settings updated.\" in res.data\n\n    # Add our URL to the import page\n    test_url = url_for(\"test_endpoint\", _external=True)\n    res = client.post(\n        url_for(\"imports.import_page\"), data={\"urls\": test_url},\n        follow_redirects=True\n    )\n    assert b\"1 Imported\" in res.data\n\n    wait_for_all_checks(client)\n    # Trigger a check\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    # set a new html text with a modified link\n    set_modified_ignore_response(datastore_path=datastore_path)\n    wait_for_all_checks(client)\n\n    # Trigger a check\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    wait_for_all_checks(client)\n\n    # We should not see the rendered anchor tag\n    res = client.get(url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"))\n    assert '(/modified_link)' not in res.data.decode()\n\n    # Goto the settings page, ENABLE render anchor tag\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\n            \"requests-time_between_check-minutes\": 180,\n            \"application-render_anchor_tag_content\": \"true\",\n            \"application-fetch_backend\": \"html_requests\",\n        },\n        follow_redirects=True,\n    )\n    assert b\"Settings updated.\" in res.data\n\n    # Trigger a check\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    # Give the thread time to pick it up\n    wait_for_all_checks(client)\n\n\n\n    # check that the anchor tag content is rendered\n    res = client.get(url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"))\n    assert '(/modified_link)' in res.data.decode()\n\n    # since the link has changed, and we chose to render anchor tag content,\n    # we should detect a change (new 'has-unread-changes' class)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b\"unviewed\" in res.data\n    assert b\"/test-endpoint\" in res.data\n\n    # Cleanup everything\n    delete_all_watches(client)\n\n"
  },
  {
    "path": "changedetectionio/tests/test_ignorestatuscode.py",
    "content": "#!/usr/bin/env python3\n\nimport time\nfrom flask import url_for\nfrom .util import live_server_setup, wait_for_all_checks\nimport os\n\n\n\n\n\ndef set_original_response(datastore_path):\n    test_return_data = \"\"\"\n       \n     Some initial text
    \n     
    Which is across multiple lines
    \n     
    \n     So let's see what happens.  
    \n     \n     \n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n\n\ndef set_some_changed_response(datastore_path):\n    test_return_data = \"\"\"\n       \n     Some initial text
    \n     
    Which is across multiple lines, and a new thing too.
    \n     
    \n     So let's see what happens.  
    \n     \n     \n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n\n\ndef test_normal_page_check_works_with_ignore_status_code(client, live_server, measure_memory_usage, datastore_path):\n    from loguru import logger\n\n    set_original_response(datastore_path=datastore_path)\n\n    # Goto the settings page, add our ignore text\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\n            \"requests-time_between_check-minutes\": 180,\n            \"application-ignore_status_codes\": \"y\",\n            'application-fetch_backend': \"html_requests\"\n        },\n        follow_redirects=True\n    )\n    assert b\"Settings updated.\" in res.data\n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n\n    logger.info(f\"TEST: First check - queuing UUID {uuid}\")\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    logger.info(f\"TEST: Waiting for first check to complete\")\n    wait_result = wait_for_all_checks(client)\n    logger.info(f\"TEST: First check wait completed: {wait_result}\")\n\n    # Check history after first check\n    watch = client.application.config.get('DATASTORE').data['watching'][uuid]\n    logger.info(f\"TEST: After first check - history count: {len(watch.history.keys())}\")\n\n    set_some_changed_response(datastore_path=datastore_path)\n\n    # Trigger a check\n    logger.info(f\"TEST: Second check - queuing UUID {uuid}\")\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    logger.info(f\"TEST: Waiting for second check to complete\")\n    wait_result = wait_for_all_checks(client)\n    logger.info(f\"TEST: Second check wait completed: {wait_result}\")\n\n    # Check history after second check\n    watch = client.application.config.get('DATASTORE').data['watching'][uuid]\n    logger.info(f\"TEST: After second check - history count: {len(watch.history.keys())}\")\n    logger.info(f\"TEST: Watch history keys: {list(watch.history.keys())}\")\n\n    # It should report nothing found (no new 'has-unread-changes' class)\n    res = client.get(url_for(\"watchlist.index\"))\n\n    if b'has-unread-changes' not in res.data:\n        logger.error(f\"TEST FAILED: has-unread-changes not found in response\")\n        logger.error(f\"TEST: Watch last_error: {watch.get('last_error')}\")\n        logger.error(f\"TEST: Watch last_checked: {watch.get('last_checked')}\")\n\n    assert b'has-unread-changes' in res.data\n    assert b'/test-endpoint' in res.data\n\n\n# Tests the whole stack works with staus codes ignored\ndef test_403_page_check_works_with_ignore_status_code(client, live_server, measure_memory_usage, datastore_path):\n\n    set_original_response(datastore_path=datastore_path)\n\n    # Give the endpoint time to spin up\n    time.sleep(1)\n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', status_code=403, _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    wait_for_all_checks(client)\n\n    # Goto the edit page, check our ignore option\n    # Add our URL to the import page\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={\"ignore_status_codes\": \"y\", \"url\": test_url, \"tags\": \"\", \"headers\": \"\", 'fetch_backend': \"html_requests\", \"time_between_check_use_default\": \"y\"},\n        follow_redirects=True\n    )\n    assert b\"Updated watch.\" in res.data\n\n    # Give the thread time to pick it up\n    wait_for_all_checks(client)\n\n    #  Make a change\n    set_some_changed_response(datastore_path=datastore_path)\n\n    # Trigger a check\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    # Give the thread time to pick it up\n    wait_for_all_checks(client)\n\n    # It should have 'has-unread-changes' still\n    # Because it should be looking at only that 'sametext' id\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'has-unread-changes' in res.data\n\n"
  },
  {
    "path": "changedetectionio/tests/test_ignorewhitespace.py",
    "content": "#!/usr/bin/env python3\n\nimport time\nfrom flask import url_for\nimport os\n\nfrom .util import live_server_setup, delete_all_watches, wait_for_all_checks\n\n\n# Should be the same as set_original_ignore_response(datastore_path=datastore_path) but with a little more whitespacing\ndef set_original_ignore_response_but_with_whitespace(datastore_path):\n    test_return_data = \"\"\"\n       \n     Some initial text
    \n     
    \n\n\n     Which is across multiple lines
    \n     
    \n     
    \n\n         So let's see what happens.  
    \n\n\n     \n     \n\n    \"\"\"\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n\n\ndef set_original_ignore_response(datastore_path):\n    test_return_data = \"\"\"\n       \n     Some initial text
    \n     
    Which is across multiple lines
    \n     
    \n     So let's see what happens.  
    \n     \n     \n\n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n\n\n\n# If there was only a change in the whitespacing, then we shouldnt have a change detected\ndef test_check_ignore_whitespace(client, live_server, measure_memory_usage, datastore_path):\n\n\n    set_original_ignore_response(datastore_path=datastore_path)\n\n    # Goto the settings page, add our ignore text\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\n            \"requests-time_between_check-minutes\": 180,\n            \"application-ignore_whitespace\": \"y\",\n            \"application-fetch_backend\": \"html_requests\"\n        },\n        follow_redirects=True\n    )\n    assert b\"Settings updated.\" in res.data\n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    wait_for_all_checks(client)\n    # Trigger a check\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    set_original_ignore_response_but_with_whitespace(datastore_path)\n    wait_for_all_checks(client)\n    # Trigger a check\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    # Give the thread time to pick it up\n    wait_for_all_checks(client)\n\n    # It should report nothing found (no new 'has-unread-changes' class)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'has-unread-changes' not in res.data\n    assert b'/test-endpoint' in res.data\n"
  },
  {
    "path": "changedetectionio/tests/test_import.py",
    "content": "#!/usr/bin/env python3\nimport io\nimport os\nimport time\n\nfrom flask import url_for\n\nfrom .util import live_server_setup, wait_for_all_checks, delete_all_watches\n\n\n# def test_setup(client, live_server, measure_memory_usage, datastore_path):\n   #  live_server_setup(live_server) # Setup on conftest per function\n\ndef test_import(client, live_server, measure_memory_usage, datastore_path):\n    # Give the endpoint time to spin up\n    wait_for_all_checks(client)\n\n    res = client.post(\n        url_for(\"imports.import_page\"),\n        data={\n            \"distill-io\": \"\",\n            \"urls\": \"\"\"https://example.com\nhttps://example.com tag1\nhttps://example.com tag1, other tag\"\"\"\n        },\n        follow_redirects=True,\n    )\n    assert b\"3 Imported\" in res.data\n    assert b\"tag1\" in res.data\n    assert b\"other tag\" in res.data\n    delete_all_watches(client)\n\n    # Clear flask alerts\n    res = client.get( url_for(\"watchlist.index\"))\n    res = client.get( url_for(\"watchlist.index\"))\n\ndef xtest_import_skip_url(client, live_server, measure_memory_usage, datastore_path):\n\n\n    # Give the endpoint time to spin up\n    time.sleep(1)\n\n    res = client.post(\n        url_for(\"imports.import_page\"),\n        data={\n            \"distill-io\": \"\",\n            \"urls\": \"\"\"https://example.com\n:ht000000broken\n\"\"\"\n        },\n        follow_redirects=True,\n    )\n    assert b\"1 Imported\" in res.data\n    assert b\"ht000000broken\" in res.data\n    assert b\"1 Skipped\" in res.data\n    delete_all_watches(client)\n    # Clear flask alerts\n    res = client.get( url_for(\"watchlist.index\"))\n\ndef test_import_distillio(client, live_server, measure_memory_usage, datastore_path):\n\n    distill_data='''\n{\n    \"client\": {\n        \"local\": 1\n    },\n    \"data\": [\n        {\n            \"name\": \"Unraid | News\",\n            \"uri\": \"https://unraid.net/blog\",\n            \"config\": \"{\\\\\"selections\\\\\":[{\\\\\"frames\\\\\":[{\\\\\"index\\\\\":0,\\\\\"excludes\\\\\":[],\\\\\"includes\\\\\":[{\\\\\"type\\\\\":\\\\\"xpath\\\\\",\\\\\"expr\\\\\":\\\\\"(//div[@id='App']/div[contains(@class,'flex')]/main[contains(@class,'relative')]/section[contains(@class,'relative')]/div[@class='container']/div[contains(@class,'flex')]/div[contains(@class,'w-full')])[1]\\\\\"}]}],\\\\\"dynamic\\\\\":true,\\\\\"delay\\\\\":2}],\\\\\"ignoreEmptyText\\\\\":true,\\\\\"includeStyle\\\\\":false,\\\\\"dataAttr\\\\\":\\\\\"text\\\\\"}\",\n            \"tags\": [\"nice stuff\", \"nerd-news\"],\n            \"content_type\": 2,\n            \"state\": 40,\n            \"schedule\": \"{\\\\\"type\\\\\":\\\\\"INTERVAL\\\\\",\\\\\"params\\\\\":{\\\\\"interval\\\\\":4447}}\",\n            \"ts\": \"2022-03-27T15:51:15.667Z\"\n        }\n    ]\n}\t\t   \n\n'''\n\n    # Give the endpoint time to spin up\n    time.sleep(1)\n    delete_all_watches(client)\n    res = client.post(\n        url_for(\"imports.import_page\"),\n        data={\n            \"distill-io\": distill_data,\n            \"urls\" : ''\n        },\n        follow_redirects=True,\n    )\n\n\n    assert b\"Unable to read JSON file, was it broken?\" not in res.data\n    assert b\"1 Imported from Distill.io\" in res.data\n\n    res = client.get( url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"))\n\n    assert b\"https://unraid.net/blog\" in res.data\n    assert b\"Unraid | News\" in res.data\n\n\n    # flask/wtforms should recode this, check we see it\n    # wtforms encodes it like id=' ,but html.escape makes it like id='\n    # - so just check it manually :(\n    #import json\n    #import html\n    #d = json.loads(distill_data)\n    # embedded_d=json.loads(d['data'][0]['config'])\n    # x=html.escape(embedded_d['selections'][0]['frames'][0]['includes'][0]['expr']).encode('utf-8')\n    assert b\"xpath:(//div[@id='App']/div[contains(@class,'flex')]/main[contains(@class,'relative')]/section[contains(@class,'relative')]/div[@class='container']/div[contains(@class,'flex')]/div[contains(@class,'w-full')])[1]\" in res.data\n\n    # did the tags work?\n    res = client.get( url_for(\"watchlist.index\"))\n\n    # check tags\n    assert b\"nice stuff\" in res.data\n    assert b\"nerd-news\" in res.data\n\n    delete_all_watches(client)\n    # Clear flask alerts\n    res = client.get(url_for(\"watchlist.index\"))\n\ndef test_import_custom_xlsx(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"Test can upload a excel spreadsheet and the watches are created correctly\"\"\"\n\n    \n\n    dirname = os.path.dirname(__file__)\n    filename = os.path.join(dirname, 'import/spreadsheet.xlsx')\n    with open(filename, 'rb') as f:\n\n        data= {\n            'file_mapping': 'custom',\n            'custom_xlsx[col_0]': '1',\n            'custom_xlsx[col_1]': '3',\n            'custom_xlsx[col_2]': '5',\n            'custom_xlsx[col_3]': '4',\n            'custom_xlsx[col_type_0]': 'title',\n            'custom_xlsx[col_type_1]': 'url',\n            'custom_xlsx[col_type_2]': 'include_filters',\n            'custom_xlsx[col_type_3]': 'interval_minutes',\n            'xlsx_file': (io.BytesIO(f.read()), 'spreadsheet.xlsx')\n        }\n\n    res = client.post(\n        url_for(\"imports.import_page\"),\n        data=data,\n        follow_redirects=True,\n    )\n\n    assert b'4 imported from custom .xlsx' in res.data\n    # Because this row was actually just a header with no usable URL, we should get an error\n    assert b'Error processing row number 1' in res.data\n\n    res = client.get(\n        url_for(\"watchlist.index\")\n    )\n\n    assert b'Somesite results ABC' in res.data\n    assert b'City news results' in res.data\n\n    # Just find one to check over\n    for uuid, watch in live_server.app.config['DATASTORE'].data['watching'].items():\n        if watch.get('title') == 'Somesite results ABC':\n            filters = watch.get('include_filters')\n            assert filters[0] == '/html[1]/body[1]/div[4]/div[1]/div[1]/div[1]||//*[@id=\\'content\\']/div[3]/div[1]/div[1]||//*[@id=\\'content\\']/div[1]'\n            assert watch.get('time_between_check') == {'weeks': 0, 'days': 1, 'hours': 6, 'minutes': 24, 'seconds': 0}\n\n    delete_all_watches(client)\n\ndef test_import_watchete_xlsx(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"Test can upload a excel spreadsheet and the watches are created correctly\"\"\"\n\n    \n    dirname = os.path.dirname(__file__)\n    filename = os.path.join(dirname, 'import/spreadsheet.xlsx')\n    with open(filename, 'rb') as f:\n\n        data= {\n            'file_mapping': 'wachete',\n            'xlsx_file': (io.BytesIO(f.read()), 'spreadsheet.xlsx')\n        }\n\n    res = client.post(\n        url_for(\"imports.import_page\"),\n        data=data,\n        follow_redirects=True,\n    )\n\n    assert b'4 imported from Wachete .xlsx' in res.data\n\n    res = client.get(\n        url_for(\"watchlist.index\")\n    )\n\n    assert b'Somesite results ABC' in res.data\n    assert b'City news results' in res.data\n\n    # Just find one to check over\n    for uuid, watch in live_server.app.config['DATASTORE'].data['watching'].items():\n        if watch.get('title') == 'Somesite results ABC':\n            filters = watch.get('include_filters')\n            assert filters[0] == '/html[1]/body[1]/div[4]/div[1]/div[1]/div[1]||//*[@id=\\'content\\']/div[3]/div[1]/div[1]||//*[@id=\\'content\\']/div[1]'\n            assert watch.get('time_between_check') == {'weeks': 0, 'days': 1, 'hours': 6, 'minutes': 24, 'seconds': 0}\n            assert watch.get('fetch_backend') == 'html_requests' # Has inactive 'dynamic wachet'\n\n        if watch.get('title') == 'JS website':\n            assert watch.get('fetch_backend') == 'html_webdriver' # Has active 'dynamic wachet'\n\n        if watch.get('title') == 'system default website':\n            assert watch.get('fetch_backend') == 'system' # uses default if blank\n\n    delete_all_watches(client)\n"
  },
  {
    "path": "changedetectionio/tests/test_jinja2.py",
    "content": "#!/usr/bin/env python3\n\nimport time\nimport arrow\nfrom flask import url_for\nfrom .util import live_server_setup, wait_for_all_checks\nfrom ..jinja2_custom import render\n\n\n# def test_setup(client, live_server, measure_memory_usage, datastore_path):\n   # #  live_server_setup(live_server) # Setup on conftest per function\n\n# If there was only a change in the whitespacing, then we shouldnt have a change detected\ndef test_jinja2_in_url_query(client, live_server, measure_memory_usage, datastore_path):\n    \n\n    # Add our URL to the import page\n    test_url = url_for('test_return_query', _external=True)\n\n    # because url_for() will URL-encode the var, but we dont here\n    full_url = \"{}?{}\".format(test_url,\n                              \"date={% now 'Europe/Berlin', '%Y' %}.{% now 'Europe/Berlin', '%m' %}.{% now 'Europe/Berlin', '%d' %}\", )\n    res = client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": full_url, \"tags\": \"test\"},\n        follow_redirects=True\n    )\n    assert b\"Watch added\" in res.data\n    wait_for_all_checks(client)\n\n    # It should report nothing found (no new 'has-unread-changes' class)\n    res = client.get(\n        url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"),\n        follow_redirects=True\n    )\n    assert b'date=2' in res.data\n\n# Test for issue #1493 - jinja2-time offset functionality\ndef test_jinja2_time_offset_in_url_query(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"Test that jinja2 time offset expressions work in watch URLs (issue #1493).\"\"\"\n\n    # Add our URL to the import page with time offset expression\n    test_url = url_for('test_return_query', _external=True)\n\n    # Test the exact syntax from issue #1493 that was broken in jinja2-time\n    # This should work now with our custom TimeExtension\n    full_url = \"{}?{}\".format(test_url,\n                              \"timestamp={% now 'utc' - 'minutes=11', '%Y-%m-%d %H:%M' %}\", )\n    res = client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": full_url, \"tags\": \"test\"},\n        follow_redirects=True\n    )\n    assert b\"Watch added\" in res.data\n    wait_for_all_checks(client)\n\n    # Verify the URL was processed correctly (should not have errors)\n    res = client.get(\n        url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"),\n        follow_redirects=True\n    )\n    # Should have a valid timestamp in the response\n    assert b'timestamp=' in res.data\n    # Should not have template error\n    assert b'Invalid template' not in res.data\n\n\n# https://techtonics.medium.com/secure-templating-with-jinja2-understanding-ssti-and-jinja2-sandbox-environment-b956edd60456\ndef test_jinja2_security_url_query(client, live_server, measure_memory_usage, datastore_path):\n    # Add our URL to the import page\n    test_url = url_for('test_return_query', _external=True)\n\n    full_url = test_url + \"?date={{ ''.__class__.__mro__[1].__subclasses__()}}\"\n\n    res = client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": full_url, \"tags\": \"test\"},\n        follow_redirects=True\n    )\n    assert b\"Watch added\" not in res.data\n\n\ndef test_timezone(mocker):\n    \"\"\"Verify that timezone is parsed.\"\"\"\n\n    timezone = 'America/Buenos_Aires'\n    currentDate = arrow.now(timezone)\n    arrowNowMock = mocker.patch(\"changedetectionio.jinja2_custom.extensions.TimeExtension.arrow.now\")\n    arrowNowMock.return_value = currentDate\n    finalRender = render(f\"{{% now '{timezone}' %}}\")\n\n    assert finalRender == currentDate.strftime('%a, %d %b %Y %H:%M:%S')\n\ndef test_format(mocker):\n    \"\"\"Verify that format is parsed.\"\"\"\n\n    timezone = 'utc'\n    format = '%d %b %Y %H:%M:%S'\n    currentDate = arrow.now(timezone)\n    arrowNowMock = mocker.patch(\"arrow.now\")\n    arrowNowMock.return_value = currentDate\n    finalRender = render(f\"{{% now '{timezone}', '{format}' %}}\")\n\n    assert finalRender == currentDate.strftime(format)\n\ndef test_add_time(environment):\n    \"\"\"Verify that added time offset can be parsed.\"\"\"\n\n    finalRender = render(\"{% now 'utc' + 'hours=2,seconds=30' %}\")\n\n    assert finalRender == \"Thu, 10 Dec 2015 01:33:31\"\n\ndef test_add_weekday(mocker):\n    \"\"\"Verify that added weekday offset can be parsed.\"\"\"\n\n    timezone = 'utc'\n    currentDate = arrow.now(timezone)\n    arrowNowMock = mocker.patch(\"changedetectionio.jinja2_custom.extensions.TimeExtension.arrow.now\")\n    arrowNowMock.return_value = currentDate\n    finalRender = render(f\"{{% now '{timezone}' + 'weekday=1' %}}\")\n\n    assert finalRender == currentDate.shift(weekday=1).strftime('%a, %d %b %Y %H:%M:%S')\n\n\ndef test_substract_time(environment):\n    \"\"\"Verify that substracted time offset can be parsed.\"\"\"\n\n    finalRender = render(\"{% now 'utc' - 'minutes=11' %}\")\n\n    assert finalRender == \"Wed, 09 Dec 2015 23:22:01\"\n\n\ndef test_offset_with_format(environment):\n    \"\"\"Verify that offset works together with datetime format.\"\"\"\n\n    finalRender = render(\n        \"{% now 'utc' - 'days=2,minutes=33,seconds=1', '%d %b %Y %H:%M:%S' %}\"\n    )\n\n    assert finalRender == \"07 Dec 2015 23:00:00\"\n\ndef test_default_timezone_empty_string(environment):\n    \"\"\"Verify that empty timezone string uses the default timezone (UTC in test environment).\"\"\"\n\n    # Empty string should use the default timezone which is 'UTC' (or from application settings)\n    finalRender = render(\"{% now '' %}\")\n\n    # Should render with default format and UTC timezone (matches environment fixture)\n    assert finalRender == \"Wed, 09 Dec 2015 23:33:01\"\n\ndef test_default_timezone_with_offset(environment):\n    \"\"\"Verify that empty timezone works with offset operations.\"\"\"\n\n    # Empty string with offset should use default timezone\n    finalRender = render(\"{% now '' + 'hours=2', '%d %b %Y %H:%M:%S' %}\")\n\n    assert finalRender == \"10 Dec 2015 01:33:01\"\n\ndef test_default_timezone_subtraction(environment):\n    \"\"\"Verify that empty timezone works with subtraction offset.\"\"\"\n\n    finalRender = render(\"{% now '' - 'minutes=11' %}\")\n\n    assert finalRender == \"Wed, 09 Dec 2015 23:22:01\"\n\ndef test_regex_replace_basic():\n    \"\"\"Test basic regex_replace functionality.\"\"\"\n\n    # Simple word replacement\n    finalRender = render(\"{{ 'hello world' | regex_replace('world', 'universe') }}\")\n    assert finalRender == \"hello universe\"\n\ndef test_regex_replace_with_groups():\n    \"\"\"Test regex_replace with capture groups (issue #3501 use case).\"\"\"\n\n    # Transform HTML table data as described in the issue\n    template = \"{{ 'thingother' | regex_replace('([^<]+)([^<]+)', 'ThingLabel: \\\\\\\\1\\\\nOtherLabel: \\\\\\\\2') }}\"\n    finalRender = render(template)\n    assert \"ThingLabel: thing\" in finalRender\n    assert \"OtherLabel: other\" in finalRender\n\ndef test_regex_replace_multiple_matches():\n    \"\"\"Test regex_replace replacing multiple occurrences.\"\"\"\n\n    finalRender = render(\"{{ 'foo bar foo baz' | regex_replace('foo', 'qux') }}\")\n    assert finalRender == \"qux bar qux baz\"\n\ndef test_regex_replace_count_parameter():\n    \"\"\"Test regex_replace with count parameter to limit replacements.\"\"\"\n\n    finalRender = render(\"{{ 'foo bar foo baz' | regex_replace('foo', 'qux', 1) }}\")\n    assert finalRender == \"qux bar foo baz\"\n\ndef test_regex_replace_empty_replacement():\n    \"\"\"Test regex_replace with empty replacement (removal).\"\"\"\n\n    finalRender = render(\"{{ 'hello world 123' | regex_replace('[0-9]+', '') }}\")\n    assert finalRender == \"hello world \"\n\ndef test_regex_replace_no_match():\n    \"\"\"Test regex_replace when pattern doesn't match.\"\"\"\n\n    finalRender = render(\"{{ 'hello world' | regex_replace('xyz', 'abc') }}\")\n    assert finalRender == \"hello world\"\n\ndef test_regex_replace_invalid_regex():\n    \"\"\"Test regex_replace with invalid regex pattern returns original value.\"\"\"\n\n    # Invalid regex (unmatched parenthesis)\n    finalRender = render(\"{{ 'hello world' | regex_replace('(invalid', 'replacement') }}\")\n    assert finalRender == \"hello world\"\n\ndef test_regex_replace_special_characters():\n    \"\"\"Test regex_replace with special regex characters.\"\"\"\n\n    finalRender = render(\"{{ 'Price: $50.00' | regex_replace('\\\\\\\\$([0-9.]+)', 'USD \\\\\\\\1') }}\")\n    assert finalRender == \"Price: USD 50.00\"\n\ndef test_regex_replace_multiline():\n    \"\"\"Test regex_replace on multiline text.\"\"\"\n\n    template = \"{{ 'line1\\\\nline2\\\\nline3' | regex_replace('^line', 'row') }}\"\n    finalRender = render(template)\n    # By default re.sub doesn't use MULTILINE flag, so only first line matches with ^\n    assert finalRender == \"row1\\nline2\\nline3\"\n\ndef test_regex_replace_with_notification_context():\n    \"\"\"Test regex_replace with notification diff variable.\"\"\"\n\n    # Simulate how it would be used in notifications with diff variable\n    from changedetectionio.notification_service import NotificationContextData\n\n    context = NotificationContextData()\n    context['diff'] = 'value1value2'\n\n    template = \"{{ diff | regex_replace('([^<]+)', '\\\\\\\\1 ') }}\"\n\n    from changedetectionio.jinja2_custom import create_jinja_env\n    from jinja2 import BaseLoader\n\n    jinja2_env = create_jinja_env(loader=BaseLoader)\n    jinja2_env.globals.update(context)\n    finalRender = jinja2_env.from_string(template).render()\n\n    assert \"value1 value2 \" in finalRender\n\ndef test_regex_replace_security_large_input():\n    \"\"\"Test regex_replace handles large input safely.\"\"\"\n\n    # Create a large input string (over 10MB)\n    large_input = \"x\" * (1024 * 1024 * 10 + 1000)\n    template = \"{{ large_input | regex_replace('x', 'y') }}\"\n\n    from changedetectionio.jinja2_custom import create_jinja_env\n    from jinja2 import BaseLoader\n\n    jinja2_env = create_jinja_env(loader=BaseLoader)\n    jinja2_env.globals['large_input'] = large_input\n    finalRender = jinja2_env.from_string(template).render()\n\n    # Should be truncated to 10MB\n    assert len(finalRender) == 1024 * 1024 * 10\n\ndef test_regex_replace_security_long_pattern():\n    \"\"\"Test regex_replace rejects very long patterns.\"\"\"\n\n    # Pattern longer than 500 chars should be rejected\n    long_pattern = \"a\" * 501\n    finalRender = render(\"{{ 'test' | regex_replace('\" + long_pattern + \"', 'replacement') }}\")\n\n    # Should return original value when pattern is too long\n    assert finalRender == \"test\"\n\ndef test_regex_replace_security_dangerous_pattern():\n    \"\"\"Test regex_replace detects and rejects dangerous nested quantifiers.\"\"\"\n\n    # Patterns that could cause catastrophic backtracking\n    dangerous_patterns = [\n        \"(a+)+\",\n        \"(a*)+\",\n        \"(a+)*\",\n        \"(a*)*\",\n    ]\n\n    for dangerous in dangerous_patterns:\n        # Create a template with the dangerous pattern\n        # Using single quotes to avoid escaping issues\n        from changedetectionio.jinja2_custom import create_jinja_env\n        from jinja2 import BaseLoader\n\n        jinja2_env = create_jinja_env(loader=BaseLoader)\n        jinja2_env.globals['pattern'] = dangerous\n        template = \"{{ 'aaaaaaaaaa' | regex_replace(pattern, 'x') }}\"\n        finalRender = jinja2_env.from_string(template).render()\n\n        # Should return original value when dangerous pattern is detected\n        assert finalRender == \"aaaaaaaaaa\"\n\ndef test_regex_replace_security_timeout_protection():\n    \"\"\"Test that regex_replace has timeout protection (if SIGALRM available).\"\"\"\n    import signal\n\n    # Only test on systems that support SIGALRM\n    if not hasattr(signal, 'SIGALRM'):\n        # Skip test on Windows and other systems without SIGALRM\n        return\n\n    # This pattern is known to cause exponential backtracking on certain inputs\n    # but should be caught by our dangerous pattern detector\n    # We're mainly testing that the timeout mechanism works\n\n    from changedetectionio.jinja2_custom import regex_replace\n\n    # Create input that could trigger slow regex\n    test_input = \"a\" * 50 + \"b\"\n\n    # This shouldn't take long due to our protections\n    result = regex_replace(test_input, \"a+b\", \"x\")\n\n    # Should complete and return a result\n    assert result is not None"
  },
  {
    "path": "changedetectionio/tests/test_jsonpath_jq_selector.py",
    "content": "#!/usr/bin/env python3\n# coding=utf-8\n\nimport time\nfrom flask import url_for\nfrom markupsafe import escape\nfrom . util import live_server_setup, wait_for_all_checks, delete_all_watches\nimport pytest\nimport os\njq_support = True\n\ntry:\n    import jq\nexcept ModuleNotFoundError:\n    jq_support = False\n\n\n\ndef test_jsonp_treated_as_plaintext():\n    from ..processors.magic import guess_stream_type\n\n    # JSONP content (server wrongly claims application/json) should be detected as plaintext\n    # Callback names are arbitrary identifiers, not always 'cb'\n    jsonp_content = 'jQuery123456({ \"version\": \"8.0.41\", \"url\": \"https://example.com/app.apk\" })'\n    result = guess_stream_type(http_content_header=\"application/json\", content=jsonp_content)\n    assert result.is_json is False\n    assert result.is_plaintext is True\n\n    # Variation with dotted callback name e.g. jQuery.cb(...)\n    jsonp_dotted = 'some.callback({ \"version\": \"1.0\" })'\n    result = guess_stream_type(http_content_header=\"application/json\", content=jsonp_dotted)\n    assert result.is_json is False\n    assert result.is_plaintext is True\n\n    # Real JSON should still be detected as JSON\n    json_content = '{ \"version\": \"8.0.41\", \"url\": \"https://example.com/app.apk\" }'\n    result = guess_stream_type(http_content_header=\"application/json\", content=json_content)\n    assert result.is_json is True\n    assert result.is_plaintext is False\n\n\ndef test_jsonp_json_filter_extraction():\n    from .. import html_tools\n\n    # Tough case: dotted namespace callback, trailing semicolon, deeply nested content with arrays\n    jsonp_content = 'weixin.update.callback({\"platforms\": {\"android\": {\"variants\": [{\"arch\": \"arm64\", \"versionName\": \"8.0.68\", \"url\": \"https://example.com/app-arm64.apk\"}, {\"arch\": \"arm32\", \"versionName\": \"8.0.41\", \"url\": \"https://example.com/app-arm32.apk\"}]}}});'\n\n    # Deep nested jsonpath filter into array element\n    text = html_tools.extract_json_as_string(jsonp_content, \"json:$.platforms.android.variants[0].versionName\")\n    assert text == '\"8.0.68\"'\n\n    # Filter that selects the second array element\n    text = html_tools.extract_json_as_string(jsonp_content, \"json:$.platforms.android.variants[1].arch\")\n    assert text == '\"arm32\"'\n\n    if jq_support:\n        text = html_tools.extract_json_as_string(jsonp_content, \"jq:.platforms.android.variants[0].versionName\")\n        assert text == '\"8.0.68\"'\n\n        text = html_tools.extract_json_as_string(jsonp_content, \"jqraw:.platforms.android.variants[1].url\")\n        assert text == \"https://example.com/app-arm32.apk\"\n\n\ndef test_unittest_inline_html_extract():\n    # So lets pretend that the JSON we want is inside some HTML\n    content=\"\"\"\n    \n    \n    food and stuff and more\n    \n    \n    \n\nand it can also be repeated\n\n
    ok
    \n\n\n\n    \"\"\"\n    from .. import html_tools\n\n    # See that we can find the second ',\n    # LD JSON\n    '',\n    ''\n    # Microdata\n    '
    Example Product
    This is a sample product description.
    Price: $$$PRICE$$
    '\n]\n\nout_of_stock_props = [\n    # out of stock AND contains multiples\n    ''\n]\n\ndef set_original_response(datastore_path, props_markup='', price=\"121.95\"):\n    props_markup=props_markup.replace('$$PRICE$$', price)\n    test_return_data = f\"\"\"\n       \n     Some initial text
    \n     
    Which is across multiple lines
    \n     
    \n     So let's see what happens.  
    \n     
    price: ${price}
    \n     {props_markup}\n     \n     \n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n    time.sleep(1)\n    return None\n\ndef test_restock_itemprop_basic(client, live_server, measure_memory_usage, datastore_path):\n\n    test_url = url_for('test_endpoint', _external=True)\n\n    # By default it should enable ('in_stock_processing') == 'all_changes'\n\n    for p in instock_props:\n        set_original_response(props_markup=p, datastore_path=datastore_path)\n        client.post(\n            url_for(\"ui.ui_views.form_quick_watch_add\"),\n            data={\"url\": test_url, \"tags\": 'restock tests', 'processor': 'restock_diff'},\n            follow_redirects=True\n        )\n        wait_for_all_checks(client)\n        res = client.get(url_for(\"watchlist.index\"))\n        assert b'more than one price detected' not in res.data\n        assert b'has-restock-info' in res.data\n        assert b' in-stock' in res.data\n        assert b' not-in-stock' not in res.data\n        delete_all_watches(client)\n\n\n    for p in out_of_stock_props:\n        set_original_response(props_markup=p, datastore_path=datastore_path)\n        client.post(\n            url_for(\"ui.ui_views.form_quick_watch_add\"),\n            data={\"url\": test_url, \"tags\": '', 'processor': 'restock_diff'},\n            follow_redirects=True\n        )\n        wait_for_all_checks(client)\n        res = client.get(url_for(\"watchlist.index\"))\n\n        assert b'has-restock-info not-in-stock' in res.data\n\n        delete_all_watches(client)\n\ndef test_itemprop_price_change(client, live_server, measure_memory_usage, datastore_path):\n    \n\n    # Out of the box 'Follow price changes' should be ON\n    test_url = url_for('test_endpoint', _external=True)\n\n    set_original_response(props_markup=instock_props[0], price=\"190.95\", datastore_path=datastore_path)\n    client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": test_url, \"tags\": 'restock tests', 'processor': 'restock_diff'},\n        follow_redirects=True\n    )\n\n    # A change in price, should trigger a change by default\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'190.95' in res.data\n\n    # basic price change, look for notification\n    set_original_response(props_markup=instock_props[0], price='180.45', datastore_path=datastore_path)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'180.45' in res.data\n    assert b'has-unread-changes' in res.data\n    client.get(url_for(\"ui.mark_all_viewed\"), follow_redirects=True)\n    time.sleep(0.2)\n\n\n    # turning off price change trigger, but it should show the new price, with no change notification\n    set_original_response(props_markup=instock_props[0], price='120.45', datastore_path=datastore_path)\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={\"processor_config_restock_diff-follow_price_changes\": \"\", \"url\": test_url, \"tags\": \"\", \"headers\": \"\", 'fetch_backend': \"html_requests\", \"time_between_check_use_default\": \"y\"},\n        follow_redirects=True\n    )\n    assert b\"Updated watch.\" in res.data\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'120.45' in res.data\n    assert b'has-unread-changes' not in res.data\n\n\n    delete_all_watches(client)\n\ndef _run_test_minmax_limit(client, extra_watch_edit_form, datastore_path):\n\n    delete_all_watches(client)\n\n    test_url = url_for('test_endpoint', _external=True)\n\n    set_original_response(props_markup=instock_props[0], price=\"950.95\", datastore_path=datastore_path)\n    client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": test_url, \"tags\": 'restock tests', 'processor': 'restock_diff'},\n        follow_redirects=True\n    )\n    wait_for_all_checks(client)\n\n    data = {\n        \"tags\": \"\",\n        \"url\": test_url,\n        \"headers\": \"\",\n        \"time_between_check-hours\": 5,\n        'fetch_backend': \"html_requests\",\n        \"time_between_check_use_default\": \"y\"\n    }\n    data.update(extra_watch_edit_form)\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data=data,\n        follow_redirects=True\n    )\n    assert b\"Updated watch.\" in res.data\n    wait_for_all_checks(client)\n\n    client.get(url_for(\"ui.mark_all_viewed\"))\n\n    # price changed to something greater than min (900), BUT less than max (1100).. should be no change\n    set_original_response(props_markup=instock_props[0], price='1000.45', datastore_path=datastore_path)\n    client.get(url_for(\"ui.form_watch_checknow\"))\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"watchlist.index\"))\n\n    assert b'more than one price detected' not in res.data\n    # BUT the new price should show, even tho its within limits\n    assert b'1,000.45' or b'1000.45' in res.data #depending on locale\n    assert b'has-unread-changes' not in res.data\n\n    # price changed to something LESS than min (900), SHOULD be a change\n    set_original_response(props_markup=instock_props[0], price='890.45', datastore_path=datastore_path)\n\n    res = client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    assert b'Queued 1 watch for rechecking.' in res.data\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'890.45' in res.data\n    assert b'has-unread-changes' in res.data\n\n    client.get(url_for(\"ui.mark_all_viewed\"))\n\n\n    # 2715 - Price detection (once it crosses the \"lower\" threshold) again with a lower price - should trigger again!\n    set_original_response(props_markup=instock_props[0], price='820.45', datastore_path=datastore_path)\n    res = client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    assert b'Queued 1 watch for rechecking.' in res.data\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'820.45' in res.data\n    assert b'has-unread-changes' in res.data\n    client.get(url_for(\"ui.mark_all_viewed\"))\n\n    # price changed to something MORE than max (1100.10), SHOULD be a change\n    set_original_response(props_markup=instock_props[0], price='1890.45', datastore_path=datastore_path)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"watchlist.index\"))\n    # Depending on the LOCALE it may be either of these (generally for US/default/etc)\n    assert b'1,890.45' in res.data or b'1890.45' in res.data\n    assert b'has-unread-changes' in res.data\n\n    delete_all_watches(client)\n\n\ndef test_restock_itemprop_minmax(client, live_server, measure_memory_usage, datastore_path):\n    \n    extras = {\n        \"processor_config_restock_diff-follow_price_changes\": \"y\",\n        \"processor_config_restock_diff-price_change_min\": 900.0,\n        \"processor_config_restock_diff-price_change_max\": 1100.10\n    }\n    _run_test_minmax_limit(client, extra_watch_edit_form=extras, datastore_path=datastore_path)\n\ndef test_restock_itemprop_with_tag(client, live_server, measure_memory_usage, datastore_path):\n    \n\n    res = client.post(\n        url_for(\"tags.form_tag_add\"),\n        data={\"name\": \"test-tag\"},\n        follow_redirects=True\n    )\n    assert b\"Tag added\" in res.data\n\n    res = client.post(\n        url_for(\"tags.form_tag_edit_submit\", uuid=\"first\"),\n        data={\"name\": \"test-tag\",\n              \"processor_config_restock_diff-follow_price_changes\": \"y\",\n              \"processor_config_restock_diff-price_change_min\": 900.0,\n              \"processor_config_restock_diff-price_change_max\": 1100.10,\n              \"overrides_watch\": \"y\", #overrides_watch should be restock_overrides_watch\n              },\n        follow_redirects=True\n    )\n\n    extras = {\n        \"tags\": \"test-tag\"\n    }\n\n    _run_test_minmax_limit(client, extra_watch_edit_form=extras,datastore_path=datastore_path)\n    delete_all_watches(client)\n\n\n\ndef test_itemprop_percent_threshold(client, live_server, measure_memory_usage, datastore_path):\n\n    delete_all_watches(client)\n\n    test_url = url_for('test_endpoint', _external=True)\n\n    set_original_response(props_markup=instock_props[0], price=\"950.95\", datastore_path=datastore_path)\n    client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": test_url, \"tags\": 'restock tests', 'processor': 'restock_diff'},\n        follow_redirects=True\n    )\n\n    # A change in price, should trigger a change by default\n    wait_for_all_checks(client)\n\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={\"processor_config_restock_diff-follow_price_changes\": \"y\",\n              \"processor_config_restock_diff-price_change_threshold_percent\": 5.0,\n              \"url\": test_url,\n              \"tags\": \"\",\n              \"headers\": \"\",\n              'fetch_backend': \"html_requests\",\n              \"time_between_check_use_default\": \"y\"\n              },\n        follow_redirects=True\n    )\n    assert b\"Updated watch.\" in res.data\n    wait_for_all_checks(client)\n\n\n    # Basic change should not trigger\n    set_original_response(props_markup=instock_props[0], price='960.45', datastore_path=datastore_path)\n    client.get(url_for(\"ui.form_watch_checknow\"))\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'960.45' in res.data\n    assert b'has-unread-changes' not in res.data\n\n    # Bigger INCREASE change than the threshold should trigger\n    set_original_response(props_markup=instock_props[0], price='1960.45', datastore_path=datastore_path)\n    client.get(url_for(\"ui.form_watch_checknow\"))\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'1,960.45' or b'1960.45' in res.data #depending on locale\n    assert b'has-unread-changes' in res.data\n\n\n    # Small decrease should NOT trigger\n    client.get(url_for(\"ui.mark_all_viewed\"))\n    set_original_response(props_markup=instock_props[0], price='1950.45', datastore_path=datastore_path)\n    client.get(url_for(\"ui.form_watch_checknow\"))\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'1,950.45' or b'1950.45' in res.data #depending on locale\n    assert b'has-unread-changes' not in res.data\n\n\n    # Re #2600 - Switch the mode to normal type and back, and see if the values stick..\n    ###################################################################################\n    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))\n\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=uuid),\n        data={\"processor_config_restock_diff-follow_price_changes\": \"y\",\n              \"processor_config_restock_diff-price_change_threshold_percent\": 5.05,\n              \"processor\": \"text_json_diff\",\n              \"url\": test_url,\n              'fetch_backend': \"html_requests\",\n              \"time_between_check_use_default\": \"y\"\n              },\n        follow_redirects=True\n    )\n    assert b\"Updated watch.\" in res.data\n    # And back again\n    live_server.app.config['DATASTORE'].data['watching'][uuid]['processor'] = 'restock_diff'\n    res = client.get(url_for(\"ui.ui_edit.edit_page\", uuid=uuid))\n    assert b'type=\"text\" value=\"5.05\"' in res.data\n\n    delete_all_watches(client)\n\n\n\ndef test_change_with_notification_values(client, live_server, measure_memory_usage, datastore_path):\n    \n\n    if os.path.isfile(os.path.join(datastore_path, \"notification.txt\")):\n        os.unlink(os.path.join(datastore_path, \"notification.txt\"))\n\n    test_url = url_for('test_endpoint', _external=True)\n    set_original_response(props_markup=instock_props[0], price='960.45', datastore_path=datastore_path)\n\n    notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')\n\n    ######################\n    # You must add a type of 'restock_diff' for its tokens to register as valid in the global settings\n    client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": test_url, \"tags\": 'restock tests', 'processor': 'restock_diff'},\n        follow_redirects=True\n    )\n\n    # A change in price, should trigger a change by default\n    wait_for_all_checks(client)\n\n    # Should see new tokens register\n    res = client.get(url_for(\"settings.settings_page\"))\n    \n    assert b'{{restock.original_price}}' in res.data\n    assert b'Original price at first check' in res.data\n\n    #####################\n    # Set this up for when we remove the notification from the watch, it should fallback with these details\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\"application-notification_urls\": notification_url,\n              \"application-notification_title\": \"title new price {{restock.price}}\",\n              \"application-notification_body\": \"new price {{restock.price}}\",\n              \"application-notification_format\": default_notification_format,\n              \"requests-time_between_check-minutes\": 180,\n              'application-fetch_backend': \"html_requests\"},\n        follow_redirects=True\n    )\n\n    # check tag accepts without error\n\n    # Check the watches in these modes add the tokens for validating\n    assert b\"A variable or function is not defined\" not in res.data\n\n    assert b\"Settings updated.\" in res.data\n\n\n    set_original_response(props_markup=instock_props[0], price='960.45', datastore_path=datastore_path)\n    # A change in price, should trigger a change by default\n    set_original_response(props_markup=instock_props[0], price='1950.45', datastore_path=datastore_path)\n    client.get(url_for(\"ui.form_watch_checknow\"))\n    wait_for_all_checks(client)\n    wait_for_notification_endpoint_output(datastore_path=datastore_path)\n    assert os.path.isfile(os.path.join(datastore_path, \"notification.txt\")), \"Notification received\"\n    with open(os.path.join(datastore_path, \"notification.txt\"), 'r') as f:\n        notification = f.read()\n        assert \"new price 1950.45\" in notification\n        assert \"title new price 1950.45\" in notification\n\n    ## Now test the \"SEND TEST NOTIFICATION\" is working\n    os.unlink(os.path.join(datastore_path, \"notification.txt\"))\n    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))\n    res = client.post(url_for(\"ui.ui_notification.ajax_callback_send_notification_test\", watch_uuid=uuid), data={}, follow_redirects=True)\n    wait_for_notification_endpoint_output(datastore_path=datastore_path)\n    assert os.path.isfile(os.path.join(datastore_path, \"notification.txt\")), \"Notification received\"\n\n    delete_all_watches(client)\n\ndef test_data_sanity(client, live_server, measure_memory_usage, datastore_path):\n    \n\n    delete_all_watches(client)\n\n    test_url = url_for('test_endpoint', _external=True)\n    test_url2 = url_for('test_endpoint2', _external=True)\n    set_original_response(props_markup=instock_props[0], price=\"950.95\", datastore_path=datastore_path)\n    client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": test_url, \"tags\": 'restock tests', 'processor': 'restock_diff'},\n        follow_redirects=True\n    )\n\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'950.95' in res.data\n\n    # Check the restock model object doesnt store the value by mistake and used in a new one\n    client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": test_url2, \"tags\": 'restock tests', 'processor': 'restock_diff'},\n        follow_redirects=True\n    )\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert str(res.data.decode()).count(\"950.95\") == 1, \"Price should only show once (for the watch added, no other watches yet)\"\n\n    ## different test, check the edit page works on an empty request result\n    delete_all_watches(client)\n\n    client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": test_url2, \"tags\": 'restock tests', 'processor': 'restock_diff'},\n        follow_redirects=True\n    )\n    wait_for_all_checks(client)\n\n    res = client.get(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"))\n    assert test_url2.encode('utf-8') in res.data\n\n    delete_all_watches(client)\n\n# All examples should give a prive of 666.66\ndef test_special_prop_examples(client, live_server, measure_memory_usage, datastore_path):\n    import glob\n    \n\n    test_url = url_for('test_endpoint', _external=True)\n    check_path = os.path.join(os.path.dirname(__file__), \"itemprop_test_examples\", \"*.txt\")\n    files = glob.glob(check_path)\n    assert files\n    for test_example_filename in files:\n        with open(test_example_filename, 'r') as example_f:\n            with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as test_f:\n                test_f.write(f\"{example_f.read()}\")\n\n            # Now fetch it and check the price worked\n            client.post(\n                url_for(\"ui.ui_views.form_quick_watch_add\"),\n                data={\"url\": test_url, \"tags\": 'restock tests', 'processor': 'restock_diff'},\n                follow_redirects=True\n            )\n            wait_for_all_checks(client)\n            res = client.get(url_for(\"watchlist.index\"))\n            assert b'ception' not in res.data\n            assert b'155.55' in res.data\n\n    delete_all_watches(client)\n\n\ndef test_itemprop_as_str(client, live_server, measure_memory_usage, datastore_path):\n\n    test_return_data = f\"\"\"\n       \n     Some initial text
    \n     
    Which is across multiple lines
    \n\n\n\n\n\n\n     \n     \n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n\n\n    test_url = url_for('test_endpoint', _external=True)\n\n    client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": test_url, \"tags\": 'restock tests', 'processor': 'restock_diff'},\n        follow_redirects=True\n    )\n\n    client.get(url_for(\"ui.form_watch_checknow\"))\n    wait_for_all_checks(client)\n\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'767.55' in res.data"
  },
  {
    "path": "changedetectionio/tests/test_rss.py",
    "content": "#!/usr/bin/env python3\nimport os\nimport time\nfrom flask import url_for\nfrom .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, extract_rss_token_from_UI, \\\n    extract_UUID_from_client, delete_all_watches\nfrom loguru import logger\nfrom ..blueprint.rss import RSS_FORMAT_TYPES\n\n\ndef set_original_cdata_xml(datastore_path):\n    test_return_data = \"\"\"\n    \n    Gizi\n    https://test.com\n    \n    \n    \n    \n    en\n    \n    \n    <![CDATA[ <img src=\"https://testsite.com/hacked.jpg\"> Hackers can access your computer ]]>\n    \n    https://testsite.com/news/12341234234\n    \n    
    The days of Terminator and The Matrix could be closer. But be positive.
    Read more link...
     ]]>\n        \n    cybernetics\n    rand corporation\n    Tue, 17 Oct 2023 15:10:00 GMT\n    1850933241\n    \n    \n    \n    \n        \n\n    \n            Some other title    \n        https://testsite.com/news/12341234236\n        \n        Some other description\n        \n        \n        \n        \n            \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n\n\n\ndef set_html_content(datastore_path, content):\n    test_return_data = f\"\"\"\n       \n     Some initial text
    \n     
    {content}
    \n     
    \n     So let's see what happens.  
    \n     \n     \n    \"\"\"\n\n    # Write as UTF-8 encoded bytes\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"wb\") as f:\n        f.write(test_return_data.encode('utf-8'))\n\n\ndef test_rss_feed_empty(client, live_server, measure_memory_usage, datastore_path):\n\n    rss_token = extract_rss_token_from_UI(client)\n\n    res = client.get(\n        url_for(\"rss.feed\", token=rss_token, _external=True),\n        follow_redirects=True\n    )\n    assert res.status_code == 200\n    assert b'xml' in res.data\n\ndef test_rss_and_token(client, live_server, measure_memory_usage, datastore_path):\n    #   #  live_server_setup(live_server) # Setup on conftest per function\n\n    set_original_response(datastore_path=datastore_path)\n    rss_token = extract_rss_token_from_UI(client)\n\n    uuid = client.application.config.get('DATASTORE').add_watch(url=url_for('test_random_content_endpoint', _external=True))\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n    set_modified_response(datastore_path=datastore_path)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    # Add our URL to the import page\n    res = client.get(\n        url_for(\"rss.feed\", token=\"bad token\", _external=True),\n        follow_redirects=True\n    )\n\n    assert b\"Access denied, bad token\" in res.data\n\n    res = client.get(\n        url_for(\"rss.feed\", token=rss_token, _external=True),\n        follow_redirects=True\n    )\n    assert b\"Access denied, bad token\" not in res.data\n    assert b\"Random content\" in res.data\n\n    delete_all_watches(client)\n\ndef test_basic_cdata_rss_markup(client, live_server, measure_memory_usage, datastore_path):\n    \n\n    set_original_cdata_xml(datastore_path)\n    # Rarely do endpoints give the right header, usually just text/xml, so we check also for  Watch'},\n        follow_redirects=True\n    )\n    assert b\"Watch added in Paused state, saving will unpause\" in res.data\n\n    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=uuid, unpause_on_save=1),\n        data={\n                \"include_filters\": \"//item/title\",\n                \"fetch_backend\": \"html_requests\",\n                \"headers\": \"\",\n                \"proxy\": \"no-proxy\",\n                \"tags\": \"\",\n                \"url\": test_url,\n                \"time_between_check_use_default\": \"y\",\n              },\n        follow_redirects=True\n    )\n    assert b\"unpaused\" in res.data\n\n    wait_for_all_checks(client)\n\n    res = client.get(\n        url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"),\n        follow_redirects=True\n    )\n    assert b'CDATA' not in res.data\n    assert b' to 
     to be sure title renders\n\n    assert b'Hackers can access your computer' in res.data # Should ONLY be selected by the xpath\n    assert b'Some other title' in res.data  # Should ONLY be selected by the xpath\n    assert b'The days of Terminator' not in res.data # Should NOT be selected by the xpath\n    assert b'Some other description' not in res.data  # Should NOT be selected by the xpath\n\n    delete_all_watches(client)\n\n\ndef test_rss_bad_chars_breaking(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"This should absolutely trigger the RSS builder to go into worst state mode\n\n    - source: prefix means no html conversion (which kinda filters out the bad stuff)\n    - Binary data\n    - Very long so that the saving is performed by Brotli (and decoded back to bytes)\n\n    Otherwise feedgen should support regular unicode\n    \"\"\"\n    \n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        ten_kb_string = \"A\" * 10_000\n        f.write(ten_kb_string)\n\n    test_url = url_for('test_endpoint', _external=True)\n    res = client.post(\n        url_for(\"imports.import_page\"),\n        data={\"urls\": \"source:\"+test_url},\n        follow_redirects=True\n    )\n    assert b\"1 Imported\" in res.data\n    wait_for_all_checks(client)\n\n    # Set the bad content\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        jpeg_bytes = \"\\xff\\xd8\\xff\\xe0\\x00\\x10XXXXXXXX\\x00\\x01\\x02\\x00\\x00\\x01\\x00\\x01\\x00\\x00\"  # JPEG header\n        jpeg_bytes += \"A\" * 10_000\n\n        f.write(jpeg_bytes)\n\n    res = client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    assert b'Queued 1 watch for rechecking.' in res.data\n    wait_for_all_checks(client)\n    rss_token = extract_rss_token_from_UI(client)\n\n    uuid = next(iter(live_server.app.config['DATASTORE'].data['watching']))\n    i=0\n    from loguru import logger\n    # Because chardet could take a long time\n    while i<=10:\n        logger.debug(f\"History was {live_server.app.config['DATASTORE'].data['watching'][uuid].history_n}..\")\n        if live_server.app.config['DATASTORE'].data['watching'][uuid].history_n ==2:\n            break\n            i+=1\n        time.sleep(2)\n    assert live_server.app.config['DATASTORE'].data['watching'][uuid].history_n == 2\n\n    # Check RSS feed is still working\n    res = client.get(\n        url_for(\"rss.feed\", uuid=uuid, token=rss_token),\n        follow_redirects=False # Important! leave this off! it should not redirect\n    )\n    assert res.status_code == 200\n\n    #assert live_server.app.config['DATASTORE'].data['watching'][uuid].history_n == 2\n    #assert live_server.app.config['DATASTORE'].data['watching'][uuid].history_n == 2\n\n\ndef test_rss_single_watch_feed(client, live_server, measure_memory_usage, datastore_path):\n\n    app_rss_token = live_server.app.config['DATASTORE'].data['settings']['application'].get('rss_access_token')\n    rss_content_format = live_server.app.config['DATASTORE'].data['settings']['application'].get('rss_content_format')\n\n    set_original_response(datastore_path=datastore_path)\n\n\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    res = client.get(\n        url_for('rss.rss_single_watch', uuid=uuid, token=app_rss_token),\n        follow_redirects=False\n    )\n\n    assert res.status_code == 400\n    assert b'not have enough history' in res.data\n\n    set_modified_response(datastore_path=datastore_path)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    res = client.get(\n        url_for('rss.rss_single_watch', uuid=uuid, token=app_rss_token),\n        follow_redirects=False\n    )\n    assert res.status_code == 200\n    import xml.etree.ElementTree as ET\n    root = ET.fromstring(res.data)\n\n    def check_formatting(expected_type, content, url):\n        logger.debug(f\"Checking formatting type {expected_type}\")\n        if expected_type == 'text':\n            assert '
    ' not in content\n            assert 'body' not in content\n            assert '(changed) Which is across multiple lines\\n'\n            assert 'modified head title had a change.' # Because it picked it up  as watch_title in default template\n        elif expected_type == 'html':\n            assert '<p>' in content\n            assert '<body>' in content\n            assert '<p>(changed) Which is across multiple lines<br>' in content\n            assert f'href=\"{url}\">modified head title had a change.</a>'\n        elif expected_type == 'htmlcolor':\n            assert '<body>' in content\n            assert ' role=\"note\" aria-label=\"Changed text\" title=\"Changed text\">Which is across multiple lines</span>' in content\n            assert f'href=\"{url}\">modified head title had a change.</a>'\n        else:\n            raise Exception(f\"Unknown type {expected_type}\")\n\n\n    item = root.findall('.//item')[0].findtext('description')\n    check_formatting(expected_type=rss_content_format, content=item, url=test_url)\n\n    # Now the default one is over, lets try all the others\n    for k in list(RSS_FORMAT_TYPES.keys()):\n        res = client.post(\n            url_for(\"settings.settings_page\"),\n            data={\"application-rss_content_format\": k},\n            follow_redirects=True\n        )\n        assert b'Settings updated' in res.data\n\n        res = client.get(\n            url_for('rss.rss_single_watch', uuid=uuid, token=app_rss_token),\n            follow_redirects=False\n        )\n        assert res.status_code == 200\n        root = ET.fromstring(res.data)\n        item = root.findall('.//item')[0].findtext('description')\n        check_formatting(expected_type=k, content=item, url=test_url)\n\n    # Test RSS entry order: Create multiple versions and verify newest appears first\n    for version in range(3, 6):  # Create versions 3, 4, 5\n        set_html_content(datastore_path, f\"Version {version} content\")\n        client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n        wait_for_all_checks(client)\n        time.sleep(0.5)  # Small delay to ensure different timestamps\n\n    # Fetch RSS feed again to verify order\n    res = client.get(\n        url_for('rss.rss_single_watch', uuid=uuid, token=app_rss_token),\n        follow_redirects=False\n    )\n    assert res.status_code == 200\n\n    # Parse RSS and check order (newest first)\n    root = ET.fromstring(res.data)\n    items = root.findall('.//item')\n    assert len(items) >= 3, f\"Expected at least 3 items, got {len(items)}\"\n\n    # Get descriptions from first 3 items\n    descriptions = []\n    for item in items[:3]:\n        desc = item.findtext('description')\n        descriptions.append(desc if desc else \"\")\n\n    # First item should contain newest change (Version 5)\n    # Note: Content may include [edit watch] links and diff markup with HTML tags\n    # Diff markup may split version numbers or keep them together depending on change type\n    assert (\">5<\" in descriptions[0] or \"Version 5\" in descriptions[0]) and \"content\" in descriptions[0], \\\n        f\"First item should show newest change, but got: {descriptions[0][:500]}\"\n\n    # Second item should contain Version 4\n    assert (\">4<\" in descriptions[1] or \"Version 4\" in descriptions[1]) and \"content\" in descriptions[1], \\\n        f\"Second item should show Version 4, but got: {descriptions[1][:500]}\"\n\n    # Third item should contain Version 3\n    assert (\">3<\" in descriptions[2] or \"Version 3\" in descriptions[2]) and \"content\" in descriptions[2], \\\n        f\"Third item should show Version 3, but got: {descriptions[2][:500]}\"\n\n"
  },
  {
    "path": "changedetectionio/tests/test_rss_group.py",
    "content": "#!/usr/bin/env python3\n\nimport time\nfrom flask import url_for\nfrom .util import live_server_setup, wait_for_all_checks, wait_for_watch_history, extract_rss_token_from_UI, get_UUID_for_tag_name, delete_all_watches\nimport os\n\n\ndef set_original_response(datastore_path):\n    test_return_data = \"\"\"<html>\n       <body>\n     Some initial text<br>\n     <p>Watch 1 content</p>\n     <p>Watch 2 content</p>\n     </body>\n     </html>\n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n    return None\n\n\ndef set_modified_response(datastore_path):\n    test_return_data = \"\"\"<html>\n       <body>\n     Some initial text<br>\n     <p>Watch 1 content MODIFIED</p>\n     <p>Watch 2 content CHANGED</p>\n     </body>\n     </html>\n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n    return None\n\n\ndef test_rss_group(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test that RSS feed for a specific tag/group shows only watches in that group\n    and displays changes correctly.\n    \"\"\"\n\n    set_original_response(datastore_path=datastore_path)\n\n    # Create a tag/group\n    res = client.post(\n        url_for(\"tags.form_tag_add\"),\n        data={\"name\": \"test-rss-group\"},\n        follow_redirects=True\n    )\n    assert b\"Tag added\" in res.data\n    assert b\"test-rss-group\" in res.data\n\n    # Get the tag UUID\n    tag_uuid = get_UUID_for_tag_name(client, name=\"test-rss-group\")\n    assert tag_uuid is not None\n\n    # Add first watch with the tag\n    test_url_1 = url_for('test_endpoint', _external=True) + \"?watch=1\"\n    res = client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": test_url_1, \"tags\": 'test-rss-group'},\n        follow_redirects=True\n    )\n    assert b\"Watch added\" in res.data\n\n    # Add second watch with the tag\n    test_url_2 = url_for('test_endpoint', _external=True) + \"?watch=2\"\n    res = client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": test_url_2, \"tags\": 'test-rss-group'},\n        follow_redirects=True\n    )\n    assert b\"Watch added\" in res.data\n\n    # Add a third watch WITHOUT the tag (should not appear in RSS)\n    test_url_3 = url_for('test_endpoint', _external=True) + \"?watch=3\"\n    res = client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": test_url_3, \"tags\": 'other-tag'},\n        follow_redirects=True\n    )\n    assert b\"Watch added\" in res.data\n\n    # Wait for initial checks to complete\n    wait_for_all_checks(client)\n\n    # Ensure initial snapshots are saved\n    assert wait_for_watch_history(client, min_history_count=1, timeout=10), \"Watches did not save initial snapshots\"\n\n    # Trigger a change\n    set_modified_response(datastore_path=datastore_path)\n\n    # Recheck all watches\n    res = client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    # Ensure all watches have sufficient history for RSS generation\n    assert wait_for_watch_history(client, min_history_count=2, timeout=10), \"Watches did not accumulate sufficient history\"\n\n    # Get RSS token\n    rss_token = extract_rss_token_from_UI(client)\n    assert rss_token is not None\n\n    # Request RSS feed for the specific tag/group using the new endpoint\n    res = client.get(\n        url_for(\"rss.rss_tag_feed\", tag_uuid=tag_uuid, token=rss_token, _external=True),\n        follow_redirects=True\n    )\n\n    # Verify response is successful\n    assert res.status_code == 200\n    assert b\"<?xml\" in res.data or b\"<rss\" in res.data\n\n    # Verify the RSS feed contains the tag name in the title\n    assert b\"test-rss-group\" in res.data\n\n    # Verify watch 1 and watch 2 are in the RSS feed (they have the tag)\n    assert b\"watch=1\" in res.data\n    assert b\"watch=2\" in res.data\n\n    # Verify watch 3 is NOT in the RSS feed (it doesn't have the tag)\n    assert b\"watch=3\" not in res.data\n\n    # Verify the changes are shown in the RSS feed\n    assert b\"MODIFIED\" in res.data or b\"CHANGED\" in res.data\n\n    # Verify it's actual RSS/XML format\n    assert b\"<rss\" in res.data or b\"<feed\" in res.data\n\n    # Test with invalid tag UUID - should return 404\n    res = client.get(\n        url_for(\"rss.rss_tag_feed\", tag_uuid=\"invalid-uuid-12345\", token=rss_token, _external=True),\n        follow_redirects=True\n    )\n    assert res.status_code == 404\n    assert b\"not found\" in res.data\n\n    # Test with invalid token - should return 403\n    res = client.get(\n        url_for(\"rss.rss_tag_feed\", tag_uuid=tag_uuid, token=\"wrong-token\", _external=True),\n        follow_redirects=True\n    )\n    assert res.status_code == 403\n    assert b\"Access denied\" in res.data\n\n    # Clean up\n    delete_all_watches(client)\n    res = client.get(url_for(\"tags.delete_all\"), follow_redirects=True)\n    assert b'All tags deleted' in res.data\n\n\ndef test_rss_group_empty_tag(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test that RSS feed for a tag with no watches returns valid but empty RSS.\n    \"\"\"\n\n    # Create a tag with no watches\n    res = client.post(\n        url_for(\"tags.form_tag_add\"),\n        data={\"name\": \"empty-tag\"},\n        follow_redirects=True\n    )\n    assert b\"Tag added\" in res.data\n\n    tag_uuid = get_UUID_for_tag_name(client, name=\"empty-tag\")\n    assert tag_uuid is not None\n\n    # Get RSS token\n    rss_token = extract_rss_token_from_UI(client)\n\n    # Request RSS feed for empty tag\n    res = client.get(\n        url_for(\"rss.rss_tag_feed\", tag_uuid=tag_uuid, token=rss_token, _external=True),\n        follow_redirects=True\n    )\n\n    # Should still return 200 with valid RSS\n    assert res.status_code == 200\n    assert b\"<?xml\" in res.data or b\"<rss\" in res.data\n    assert b\"empty-tag\" in res.data\n\n    # Clean up\n    res = client.get(url_for(\"tags.delete_all\"), follow_redirects=True)\n    assert b'All tags deleted' in res.data\n\n\ndef test_rss_group_only_unviewed(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test that RSS feed for a tag only shows unviewed watches.\n    \"\"\"\n\n    set_original_response(datastore_path=datastore_path)\n\n    # Create a tag\n    res = client.post(\n        url_for(\"tags.form_tag_add\"),\n        data={\"name\": \"unviewed-test\"},\n        follow_redirects=True\n    )\n    assert b\"Tag added\" in res.data\n\n    tag_uuid = get_UUID_for_tag_name(client, name=\"unviewed-test\")\n\n    # Add two watches with the tag\n    test_url_1 = url_for('test_endpoint', _external=True) + \"?unviewed=1\"\n    res = client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": test_url_1, \"tags\": 'unviewed-test'},\n        follow_redirects=True\n    )\n    assert b\"Watch added\" in res.data\n\n    test_url_2 = url_for('test_endpoint', _external=True) + \"?unviewed=2\"\n    res = client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": test_url_2, \"tags\": 'unviewed-test'},\n        follow_redirects=True\n    )\n    assert b\"Watch added\" in res.data\n\n    wait_for_all_checks(client)\n    assert wait_for_watch_history(client, min_history_count=1, timeout=10), \"Initial snapshots not saved\"\n\n    # Trigger changes\n    set_modified_response(datastore_path=datastore_path)\n    res = client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n    assert wait_for_watch_history(client, min_history_count=2, timeout=10), \"History not accumulated\"\n\n    # Get RSS token\n    rss_token = extract_rss_token_from_UI(client)\n\n    # Request RSS feed - should show both watches (both unviewed)\n    res = client.get(\n        url_for(\"rss.rss_tag_feed\", tag_uuid=tag_uuid, token=rss_token, _external=True),\n        follow_redirects=True\n    )\n    assert res.status_code == 200\n    assert b\"unviewed=1\" in res.data\n    assert b\"unviewed=2\" in res.data\n\n    # Mark all as viewed\n    res = client.get(url_for('ui.mark_all_viewed'), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    # Request RSS feed again - should be empty now (no unviewed watches)\n    res = client.get(\n        url_for(\"rss.rss_tag_feed\", tag_uuid=tag_uuid, token=rss_token, _external=True),\n        follow_redirects=True\n    )\n    assert res.status_code == 200\n    # Should not contain the watch URLs anymore since they're viewed\n    assert b\"unviewed=1\" not in res.data\n    assert b\"unviewed=2\" not in res.data\n\n    # Clean up\n    delete_all_watches(client)\n    res = client.get(url_for(\"tags.delete_all\"), follow_redirects=True)\n    assert b'All tags deleted' in res.data\n"
  },
  {
    "path": "changedetectionio/tests/test_rss_reader_mode.py",
    "content": "#!/usr/bin/env python3\n\nimport time\nimport os\n\nfrom flask import url_for\nfrom .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, extract_rss_token_from_UI, \\\n    extract_UUID_from_client, delete_all_watches\n\ndef set_xmlns_purl_content(datastore_path, extra=\"\"):\n    data=f\"\"\"<rss xmlns:content=\"http://purl.org/rss/1.0/modules/content/\" xmlns:dc=\"https://purl.org/dc/elements/1.1/\" xmlns:media=\"http://search.yahoo.com/mrss/\" xmlns:atom=\"http://www.w3.org/2005/Atom\" version=\"2.0\">\n<channel>\n<atom:link href=\"https://www.xxxxxxxtechxxxxx.com/feeds.xml\" rel=\"self\" type=\"application/rss+xml\"/>\n<title>\n<![CDATA[ Latest from xxxxxxxtechxxxxx ]]>\n\nhttps://www.xxxxx.com\n\n\n\nWed, 19 Nov 2025 15:00:00 +0000\nen\n\n\n<![CDATA[ Sony Xperia 1 VII review: has Sony’s long-standing Xperia family lost what it takes to compete? ]]>\n\n\n
    On the plus side, you don't technically need to solve the final one, as you'll be able to answer that one by a process of elimination. What's more, you can make up to four mistakes, which gives you a little bit of breathing room.
    It's a little more involved than something like Wordle, however, and there are plenty of opportunities for the game to trip you up with tricks. For instance, watch out for homophones and other word games that could disguise the answers.
    It's playable for free via the NYT Games site on desktop or mobile.
     ]]>\n\nhttps://www.xxxxxxx.com/gaming/nyt-connections-today-answers-hints-20-november-2025\n\n\n\nN2C2T6DztpWdxSdKpSUx89\n\nWed, 19 Nov 2025 15:00:00 +0000\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n    \n    \n            \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(data)\n\n\n\n\ndef set_original_cdata_xml(datastore_path):\n    test_return_data = \"\"\"\n\nSecurity Bulletins on wetscale\nhttps://wetscale.com/security-bulletins/\nRecent security bulletins from wetscale\nFri, 10 Oct 2025 14:58:11 GMT\nhttps://validator.w3.org/feed/docs/rss2.html\nwetscale.com\nen-US\n© 2025 wetscale Inc. All rights reserved.\n\n\nTS-2025-005\nhttps://wetscale.com/security-bulletins/#ts-2025-005\nhttps://wetscale.com/security-bulletins/#ts-2025-005\nThu, 07 Aug 2025 00:00:00 GMT\n
    Wet noodles escape
    they also found themselves outside
         \n    \n\n\n\nTS-2025-004\nhttps://wetscale.com/security-bulletins/#ts-2025-004\nhttps://wetscale.com/security-bulletins/#ts-2025-004\nTue, 27 May 2025 00:00:00 GMT\n\n    
    The days of Terminator and The Matrix could be closer. But be positive.
    Read more link...
     ]]>\n    \n    \n        \n        \n            \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n\n\n\ndef test_rss_reader_mode(client, live_server, measure_memory_usage, datastore_path):\n    set_original_cdata_xml(datastore_path=datastore_path)\n\n    # Rarely do endpoints give the right header, usually just text/xml, so we check also for ' not in snapshot_contents\n    assert '<' not in snapshot_contents\n    assert 'The days of Terminator and The Matrix' in snapshot_contents\n    assert 'PubDate: Thu, 07 Aug 2025 00:00:00 GMT' in snapshot_contents\n    delete_all_watches(client)\n\ndef test_rss_reader_mode_with_css_filters(client, live_server, measure_memory_usage, datastore_path):\n    set_original_cdata_xml(datastore_path=datastore_path)\n\n    # Rarely do endpoints give the right header, usually just text/xml, so we check also for ' not in snapshot_contents\n    assert '<' not in snapshot_contents\n    assert 'The days of Terminator and The Matrix' in snapshot_contents\n    delete_all_watches(client)\n\n\ndef test_xmlns_purl_content(client, live_server, measure_memory_usage, datastore_path):\n    set_xmlns_purl_content(datastore_path=datastore_path)\n\n    # Rarely do endpoints give the right header, usually just text/xml, so we check also for \n           \n         
    Version {version} content
    \n         \n         \n        \"\"\"\n        with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n            f.write(test_return_data)\n\n    # Start with version 1\n    set_response(datastore_path, 1)\n\n    # Add a watch\n    test_url = url_for('test_endpoint', _external=True) + \"?order_test=1\"\n    res = client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": test_url, \"tags\": 'test-tag'},\n        follow_redirects=True\n    )\n    assert b\"Watch added\" in res.data\n\n    # Get the watch UUID\n    watch_uuid = extract_UUID_from_client(client)\n\n    # Wait for initial check\n    wait_for_all_checks(client)\n\n    # Create multiple versions by triggering changes\n    for version in range(2, 6):  # Create versions 2, 3, 4, 5\n        set_response(datastore_path, version)\n        res = client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n        wait_for_all_checks(client)\n        time.sleep(0.5)  # Small delay to ensure different timestamps\n\n    # Get RSS token\n    rss_token = extract_rss_token_from_UI(client)\n\n    # Request RSS feed for the single watch\n    res = client.get(\n        url_for(\"rss.rss_single_watch\", uuid=watch_uuid, token=rss_token, _external=True),\n        follow_redirects=True\n    )\n\n    # Should return valid RSS\n    assert res.status_code == 200\n    assert b\"= 3, f\"Expected at least 3 items, got {len(items)}\"\n\n    # Get the descriptions/content from first 3 items\n    descriptions = []\n    for item in items[:3]:\n        # Try RSS format first\n        desc = item.findtext('description')\n        if not desc:\n            # Try Atom format\n            content_elem = item.find('{http://www.w3.org/2005/Atom}content')\n            if content_elem is not None:\n                desc = content_elem.text\n        descriptions.append(desc if desc else \"\")\n\n    print(f\"First item content (first 200 chars): {descriptions[0][:200] if descriptions[0] else 'None'}\")\n    print(f\"Second item content (first 200 chars): {descriptions[1][:200] if descriptions[1] else 'None'}\")\n    print(f\"Third item content (first 200 chars): {descriptions[2][:200] if descriptions[2] else 'None'}\")\n\n    # The FIRST item should contain the NEWEST change (Version 5)\n    # The SECOND item should contain Version 4\n    # The THIRD item should contain Version 3\n    # Note: Content may include [edit watch] links and diff markup like \"(added) 5\"\n    # So we check for \"5 content\" which appears in \"Version 5 content\"\n    assert \"5 content\" in descriptions[0], \\\n        f\"First item should show newest change (with '5 content'), but got: {descriptions[0][:500]}\"\n\n    # Verify the order is correct\n    assert \"4 content\" in descriptions[1], \\\n        f\"Second item should show Version 4 (with '4 content'), but got: {descriptions[1][:500]}\"\n\n    assert \"3 content\" in descriptions[2], \\\n        f\"Third item should show Version 3 (with '3 content'), but got: {descriptions[2][:500]}\"\n\n    # Clean up\n    delete_all_watches(client)\n\n\ndef test_rss_categories_from_tags(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test that RSS feeds include category tags from watch tags.\n    \"\"\"\n\n    # Create initial content\n    test_return_data = \"\"\"\n       \n     
    Test content for RSS categories
    \n     \n     \n    \"\"\"\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n\n    # Create some tags first\n    res = client.post(\n        url_for(\"tags.form_tag_add\"),\n        data={\"name\": \"Security\"},\n        follow_redirects=True\n    )\n\n    res = client.post(\n        url_for(\"tags.form_tag_add\"),\n        data={\"name\": \"Python\"},\n        follow_redirects=True\n    )\n\n    res = client.post(\n        url_for(\"tags.form_tag_add\"),\n        data={\"name\": \"Tech News\"},\n        follow_redirects=True\n    )\n\n    # Add a watch with tags\n    test_url = url_for('test_endpoint', _external=True) + \"?category_test=1\"\n    res = client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": test_url, \"tags\": \"Security, Python, Tech News\"},\n        follow_redirects=True\n    )\n    assert b\"Watch added\" in res.data\n\n    # Get the watch UUID\n    watch_uuid = extract_UUID_from_client(client)\n\n    # Wait for initial check\n    wait_for_all_checks(client)\n\n    # Trigger one change\n    test_return_data_v2 = \"\"\"\n       \n     
    Updated content for RSS categories
    \n     \n     \n    \"\"\"\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data_v2)\n\n    res = client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    # Get RSS token\n    rss_token = extract_rss_token_from_UI(client)\n\n    # Test 1: Check single watch RSS feed\n    res = client.get(\n        url_for(\"rss.rss_single_watch\", uuid=watch_uuid, token=rss_token, _external=True),\n        follow_redirects=True\n    )\n    assert res.status_code == 200\n    assert b\"= 1, \"Expected at least 1 item in RSS feed\"\n\n    # Get categories from first item\n    categories = [cat.text for cat in items[0].findall('category')]\n\n    print(f\"Found categories in single watch RSS: {categories}\")\n\n    # Should have all three categories\n    assert \"Security\" in categories, f\"Expected 'Security' category, got: {categories}\"\n    assert \"Python\" in categories, f\"Expected 'Python' category, got: {categories}\"\n    assert \"Tech News\" in categories, f\"Expected 'Tech News' category, got: {categories}\"\n    assert len(categories) == 3, f\"Expected 3 categories, got {len(categories)}: {categories}\"\n\n    # Test 2: Check main RSS feed\n    res = client.get(\n        url_for(\"rss.feed\", token=rss_token, _external=True),\n        follow_redirects=True\n    )\n    assert res.status_code == 200\n\n    root = ET.fromstring(res.data)\n    items = root.findall('.//item')\n    assert len(items) >= 1, \"Expected at least 1 item in main RSS feed\"\n\n    # Get categories from first item in main feed\n    categories = [cat.text for cat in items[0].findall('category')]\n\n    print(f\"Found categories in main RSS feed: {categories}\")\n\n    # Should have all three categories\n    assert \"Security\" in categories, f\"Expected 'Security' category in main feed, got: {categories}\"\n    assert \"Python\" in categories, f\"Expected 'Python' category in main feed, got: {categories}\"\n    assert \"Tech News\" in categories, f\"Expected 'Tech News' category in main feed, got: {categories}\"\n\n    # Test 3: Check tag-specific RSS feed (should also have categories)\n    # Get the tag UUID for \"Security\" and verify the tag feed also has categories\n    from .util import get_UUID_for_tag_name\n    security_tag_uuid = get_UUID_for_tag_name(client, name=\"Security\")\n\n    if security_tag_uuid:\n        res = client.get(\n            url_for(\"rss.rss_tag_feed\", tag_uuid=security_tag_uuid, token=rss_token, _external=True),\n            follow_redirects=True\n        )\n        assert res.status_code == 200\n\n        root = ET.fromstring(res.data)\n        items = root.findall('.//item')\n\n        if len(items) >= 1:\n            categories = [cat.text for cat in items[0].findall('category')]\n            print(f\"Found categories in tag RSS feed: {categories}\")\n\n            # Should still have all three categories\n            assert \"Security\" in categories, f\"Expected 'Security' category in tag feed, got: {categories}\"\n            assert \"Python\" in categories, f\"Expected 'Python' category in tag feed, got: {categories}\"\n            assert \"Tech News\" in categories, f\"Expected 'Tech News' category in tag feed, got: {categories}\"\n\n    # Clean up\n    delete_all_watches(client)\n\n\n# RSS  should follow Main Settings -> Tag/Group -> Watch in that order of priority if set.\ndef test_rss_single_watch_follow_notification_body(client, live_server, measure_memory_usage, datastore_path):\n    rss_token = extract_rss_token_from_UI(client)\n\n\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\n              \"application-fetch_backend\": \"html_requests\",\n              \"application-minutes_between_check\": 180,\n              \"application-notification_body\": 'Boo yeah hello from main settings notification body
    \\nTitle: {{ watch_title }} changed',\n              \"application-notification_format\": default_notification_format,\n              \"application-rss_template_type\" : 'notification_body',\n              \"application-notification_urls\": \"\",\n\n              },\n        follow_redirects=True\n    )\n    assert b'Settings updated' in res.data\n\n\n    set_original_response(datastore_path=datastore_path)\n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url, tag=\"RSS-Custom\")\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    set_modified_response(datastore_path=datastore_path)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n\n    # Request RSS feed for the single watch\n    res = client.get(\n        url_for(\"rss.rss_single_watch\", uuid=uuid, token=rss_token, _external=True),\n        follow_redirects=True\n    )\n\n    # Should return valid RSS\n    assert res.status_code == 200\n    assert b\"\n    head title\n    \n     Some initial text
    \n     
    Which is across multiple lines
    \n     
    \n     So let's see what happens.  
    \n     \n     \n     \n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n    return None\n\n\ndef test_favicon(client, live_server, measure_memory_usage, datastore_path):\n    # Attempt to fetch it, make sure that works\n    SVG_BASE64 = 'PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAxIDEiLz4='\n    uuid = client.application.config.get('DATASTORE').add_watch(url='https://localhost')\n    live_server.app.config['DATASTORE'].data['watching'][uuid].bump_favicon(url=\"favicon-set-type.svg\",\n                                                                            favicon_base_64=SVG_BASE64\n                                                                            )\n\n\n    res = client.get(url_for('static_content', group='favicon', filename=uuid))\n    assert res.status_code == 200\n    assert len(res.data) > 10\n\n    res = client.get(url_for('static_content', group='..', filename='__init__.py'))\n    assert res.status_code != 200\n\n\n    res = client.get(url_for('static_content', group='.', filename='../__init__.py'))\n    assert res.status_code != 200\n\n    # Traverse by filename protection\n    res = client.get(url_for('static_content', group='js', filename='../styles/styles.css'))\n    assert res.status_code != 200\n\ndef test_bad_access(client, live_server, measure_memory_usage, datastore_path):\n\n    res = client.post(\n        url_for(\"imports.import_page\"),\n        data={\"urls\": 'https://localhost'},\n        follow_redirects=True\n    )\n\n    assert b\"1 Imported\" in res.data\n    wait_for_all_checks(client)\n\n    # Attempt to add a body with a GET method\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={\n              \"url\": 'javascript:alert(document.domain)',\n              \"tags\": \"\",\n              \"method\": \"GET\",\n              \"fetch_backend\": \"html_requests\",\n              \"body\": \"\",\n              \"time_between_check_use_default\": \"y\"},\n        follow_redirects=True\n    )\n\n    assert b'Watch protocol is not permitted or invalid URL format' in res.data\n\n    res = client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": '            javascript:alert(123)', \"tags\": ''},\n        follow_redirects=True\n    )\n\n    assert b'Watch protocol is not permitted or invalid URL format' in res.data\n\n    res = client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": '%20%20%20javascript:alert(123)%20%20', \"tags\": ''},\n        follow_redirects=True\n    )\n\n    assert b'Watch protocol is not permitted or invalid URL format' in res.data\n\n\n    res = client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": ' source:javascript:alert(document.domain)', \"tags\": ''},\n        follow_redirects=True\n    )\n\n    assert b'Watch protocol is not permitted or invalid URL format' in res.data\n\n    res = client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": 'https://i-wanna-xss-you.com?hereis=', \"tags\": ''},\n        follow_redirects=True\n    )\n\n    assert b'Watch protocol is not permitted or invalid URL format' in res.data\n\ndef _runner_test_various_file_slash(client, file_uri):\n\n    client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": file_uri, \"tags\": ''},\n        follow_redirects=True\n    )\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"watchlist.index\"))\n\n    substrings = [b\"URLs with hostname components are not permitted\", b\"No connection adapters were found for\"]\n\n\n    # If it is enabled at test time\n    if strtobool(os.getenv('ALLOW_FILE_URI', 'false')):\n        if file_uri.startswith('file:///'):\n            # This one should be the full qualified path to the file and should get the contents of this file\n            res = client.get(\n                url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"),\n                follow_redirects=True\n            )\n            assert b'_runner_test_various_file_slash' in res.data\n        else:\n            # This will give some error from requests or if it went to chrome, will give some other error :-)\n            assert any(s in res.data for s in substrings)\n\n    delete_all_watches(client)\n\ndef test_file_slash_access(client, live_server, measure_memory_usage, datastore_path):\n    \n\n    # file: is NOT permitted by default, so it will be caught by ALLOW_FILE_URI check\n\n    test_file_path = os.path.abspath(__file__)\n    _runner_test_various_file_slash(client, file_uri=f\"file://{test_file_path}\")\n#    _runner_test_various_file_slash(client, file_uri=f\"file:/{test_file_path}\")\n#    _runner_test_various_file_slash(client, file_uri=f\"file:{test_file_path}\") # CVE-2024-56509\n\ndef test_xss(client, live_server, measure_memory_usage, datastore_path):\n    \n    from changedetectionio.notification import (\n        default_notification_format\n    )\n    # the template helpers were named .jinja which meant they were not having jinja2 autoescape enabled.\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\"application-notification_urls\": '\">',\n              \"application-notification_title\": '\">',\n              \"application-notification_body\": '\">',\n              \"application-notification_format\": default_notification_format,\n              \"requests-time_between_check-minutes\": 180,\n              'application-fetch_backend': \"html_requests\"},\n        follow_redirects=True\n    )\n\n    assert b\"',\n            \"url\": url_for('test_endpoint', _external=True),\n            'fetch_backend': \"html_requests\",\n            \"time_between_check_use_default\": \"y\"\n        },\n        follow_redirects=True\n    )\n    assert b\"Updated watch.\" in res.data\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"watchlist.index\"))\n\n    assert b\"\" not in res.data  # this text should be there\n    assert b'<a href="https://foobar"></a><script>alert(123);</script>' in res.data\n    assert b\"https://foobar\" in res.data # this text should be there\n\n\ndef test_login_redirect_safe_urls(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test that safe redirect URLs work correctly in login flow.\n    This verifies the fix for open redirect vulnerabilities while maintaining\n    legitimate redirect functionality for both authenticated and unauthenticated users.\n    \"\"\"\n\n    # Test 1: Accessing /login?redirect=/settings when not logged in\n    # Should show the login form with redirect parameter preserved\n    res = client.get(\n        url_for(\"login\", redirect=\"/settings\"),\n        follow_redirects=False\n    )\n    # Should show login form\n    assert res.status_code == 200\n    # Check that the redirect is preserved in the hidden form field\n    assert b'name=\"redirect\"' in res.data\n\n    # Test 2: Valid internal redirect with query parameters\n    res = client.get(\n        url_for(\"login\", redirect=\"/settings?tab=notifications\"),\n        follow_redirects=False\n    )\n    assert res.status_code == 200\n    # Check that the redirect is preserved\n    assert b'value=\"/settings?tab=notifications\"' in res.data\n\n    # Test 3: Malicious external URL should be blocked and default to watchlist\n    res = client.get(\n        url_for(\"login\", redirect=\"https://evil.com/phishing\"),\n        follow_redirects=False\n    )\n    # Should show login form\n    assert res.status_code == 200\n    # The redirect parameter in the form should NOT contain the evil URL\n    # Check the actual input value, not just anywhere in the page\n    assert b'value=\"https://evil.com' not in res.data\n    assert b'value=\"/evil.com' not in res.data\n    assert b'name=\"redirect\"' in res.data\n\n    # Test 4: Double-slash attack should be blocked\n    res = client.get(\n        url_for(\"login\", redirect=\"//evil.com\"),\n        follow_redirects=False\n    )\n    assert res.status_code == 200\n    # Should not have the malicious URL in the redirect input value\n    assert b'value=\"//evil.com\"' not in res.data\n\n    # Test 5: Protocol handler exploit should be blocked\n    res = client.get(\n        url_for(\"login\", redirect=\"javascript:alert(document.domain)\"),\n        follow_redirects=False\n    )\n    assert res.status_code == 200\n    # Should not have javascript: in the redirect input value\n    assert b'value=\"javascript:' not in res.data\n\n    # Test 6: At-symbol obfuscation attack should be blocked\n    res = client.get(\n        url_for(\"login\", redirect=\"//@evil.com\"),\n        follow_redirects=False\n    )\n    assert res.status_code == 200\n    # Should not have the malicious URL in the redirect input value\n    assert b'value=\"//@evil.com\"' not in res.data\n\n    # Test 7: Multiple slashes attack should be blocked\n    res = client.get(\n        url_for(\"login\", redirect=\"////evil.com\"),\n        follow_redirects=False\n    )\n    assert res.status_code == 200\n    # Should not have the malicious URL in the redirect input value\n    assert b'value=\"////evil.com\"' not in res.data\n\n\ndef test_login_redirect_with_password(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test that redirect functionality works correctly when a password is set.\n    This ensures that notifications can always link to /login and users will\n    be redirected to the correct page after authentication.\n    \"\"\"\n\n    # Set a password\n    from changedetectionio import store\n    import base64\n    import hashlib\n\n    # Generate a test password\n    password = \"test123\"\n    salt = os.urandom(32)\n    key = hashlib.pbkdf2_hmac('sha256', password.encode('utf-8'), salt, 100000)\n    salted_pass = base64.b64encode(salt + key).decode('ascii')\n\n    # Set the password in the datastore\n    client.application.config['DATASTORE'].data['settings']['application']['password'] = salted_pass\n\n    # Test 1: Try to access /login?redirect=/settings without being logged in\n    # Should show login form and preserve redirect parameter\n    res = client.get(\n        url_for(\"login\", redirect=\"/settings\"),\n        follow_redirects=False\n    )\n    assert res.status_code == 200\n    assert b\"Password\" in res.data\n    # Check that redirect parameter is preserved in the form\n    assert b'name=\"redirect\"' in res.data\n    assert b'value=\"/settings\"' in res.data\n\n    # Test 2: Submit correct password with redirect parameter\n    # Should redirect to /settings after successful login\n    res = client.post(\n        url_for(\"login\"),\n        data={\"password\": password, \"redirect\": \"/settings\"},\n        follow_redirects=True\n    )\n    assert res.status_code == 200\n    # Should be on settings page\n    assert b\"Settings\" in res.data or b\"settings\" in res.data\n\n    # Test 3: Now that we're logged in, accessing /login?redirect=/settings\n    # should redirect immediately without showing login form\n    res = client.get(\n        url_for(\"login\", redirect=\"/\"),\n        follow_redirects=True\n    )\n    assert res.status_code == 200\n    assert b\"Already logged in\" in res.data\n\n    # Test 4: Malicious redirect should be blocked even with correct password\n    res = client.post(\n        url_for(\"login\"),\n        data={\"password\": password, \"redirect\": \"https://evil.com\"},\n        follow_redirects=True\n    )\n    # Should redirect to watchlist index instead of evil.com\n    assert b\"evil.com\" not in res.data\n\n    # Logout for cleanup\n    client.get(url_for(\"logout\"))\n\n    # Test 5: Incorrect password with redirect should stay on login page\n    res = client.post(\n        url_for(\"login\"),\n        data={\"password\": \"wrongpassword\", \"redirect\": \"/settings\"},\n        follow_redirects=True\n    )\n    assert res.status_code == 200\n    assert b\"Incorrect password\" in res.data or b\"password\" in res.data\n\n    # Clear the password\n    del client.application.config['DATASTORE'].data['settings']['application']['password']\n\n\ndef test_login_redirect_from_protected_page(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test the complete redirect flow: accessing a protected page while logged out\n    should redirect to login with the page URL, then redirect back after login.\n    This is the real-world scenario where users try to access /edit/uuid or /settings\n    and need to login first.\n    \"\"\"\n    import base64\n    import hashlib\n\n    # Add a watch first\n    set_original_response(datastore_path=datastore_path)\n    res = client.post(\n        url_for(\"imports.import_page\"),\n        data={\"urls\": url_for('test_endpoint', _external=True)},\n        follow_redirects=True\n    )\n    assert b\"1 Imported\" in res.data\n    wait_for_all_checks(client)\n\n    # Set a password\n    password = \"test123\"\n    salt = os.urandom(32)\n    key = hashlib.pbkdf2_hmac('sha256', password.encode('utf-8'), salt, 100000)\n    salted_pass = base64.b64encode(salt + key).decode('ascii')\n    client.application.config['DATASTORE'].data['settings']['application']['password'] = salted_pass\n\n    # Logout to ensure we're not authenticated\n    client.get(url_for(\"logout\"))\n\n    # Try to access a protected page (edit page for first watch)\n    res = client.get(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        follow_redirects=False\n    )\n\n    # Should redirect to login with the edit page as redirect parameter\n    assert res.status_code in [302, 303]\n    assert '/login' in res.location\n    assert 'redirect=' in res.location or 'redirect=%2F' in res.location\n\n    # Follow the redirect to login page\n    res = client.get(res.location, follow_redirects=False)\n    assert res.status_code == 200\n    assert b'Password' in res.data\n\n    # The redirect parameter should be preserved in the login form\n    # It should contain the edit page URL\n    assert b'name=\"redirect\"' in res.data\n    assert b'value=\"/edit/first\"' in res.data or b'value=\"%2Fedit%2Ffirst\"' in res.data\n\n    # Now login with correct password and the redirect parameter\n    res = client.post(\n        url_for(\"login\"),\n        data={\"password\": password, \"redirect\": \"/edit/first\"},\n        follow_redirects=False\n    )\n\n    # Should redirect to the edit page\n    assert res.status_code in [302, 303]\n    assert '/edit/first' in res.location\n\n    # Follow the redirect to verify we're on the edit page\n    res = client.get(res.location, follow_redirects=True)\n    assert res.status_code == 200\n    # Should see edit page content\n    assert b'Edit' in res.data or b'Watching' in res.data\n\n    # Cleanup\n    client.get(url_for(\"logout\"))\n    del client.application.config['DATASTORE'].data['settings']['application']['password']\n\n\ndef test_logout_with_redirect(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test that logout preserves the current page URL, so after re-login\n    the user returns to where they were before logging out.\n    Example: User is on /edit/uuid, clicks logout, then logs back in and\n    returns to /edit/uuid.\n    \"\"\"\n    import base64\n    import hashlib\n\n    # Set a password and login\n    password = \"test123\"\n    salt = os.urandom(32)\n    key = hashlib.pbkdf2_hmac('sha256', password.encode('utf-8'), salt, 100000)\n    salted_pass = base64.b64encode(salt + key).decode('ascii')\n    client.application.config['DATASTORE'].data['settings']['application']['password'] = salted_pass\n\n    # Login\n    res = client.post(\n        url_for(\"login\"),\n        data={\"password\": password},\n        follow_redirects=True\n    )\n    assert res.status_code == 200\n\n    # Now logout with a redirect parameter (simulating logout from /settings)\n    res = client.get(\n        url_for(\"logout\", redirect=\"/settings\"),\n        follow_redirects=False\n    )\n\n    # Should redirect to login with the redirect parameter\n    assert res.status_code in [302, 303]\n    assert '/login' in res.location\n    assert 'redirect=' in res.location or 'redirect=%2F' in res.location\n\n    # Follow the redirect to login page\n    res = client.get(res.location, follow_redirects=False)\n    assert res.status_code == 200\n    assert b'Password' in res.data\n    # The redirect parameter should be preserved\n    assert b'value=\"/settings\"' in res.data or b'value=\"%2Fsettings\"' in res.data\n\n    # Login again with the redirect\n    res = client.post(\n        url_for(\"login\"),\n        data={\"password\": password, \"redirect\": \"/settings\"},\n        follow_redirects=False\n    )\n\n    # Should redirect back to settings\n    assert res.status_code in [302, 303]\n    assert '/settings' in res.location or 'settings' in res.location\n\n    # Cleanup\n    del client.application.config['DATASTORE'].data['settings']['application']['password']\n\n\ndef test_static_directory_traversal(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test that the static file serving route properly blocks directory traversal attempts.\n    This tests the fix for GHSA-9jj8-v89v-xjvw (CVE pending).\n\n    The vulnerability was in /static// where the sanitization regex\n    allowed dots, enabling \"../\" traversal to read application source files.\n\n    The fix changed the regex from r'[^\\w.-]+' to r'[^a-z0-9_]+' which blocks dots.\n    \"\"\"\n\n    # Test 1: Direct .. traversal attempt (URL-encoded)\n    res = client.get(\n        \"/static/%2e%2e/flask_app.py\",\n        follow_redirects=False\n    )\n    # Should be blocked (404 or 403)\n    assert res.status_code in [404, 403], f\"Expected 404/403, got {res.status_code}\"\n    # Should NOT contain application source code\n    assert b\"def static_content\" not in res.data\n    assert b\"changedetection_app\" not in res.data\n\n    # Test 2: Direct .. traversal attempt (unencoded)\n    res = client.get(\n        \"/static/../flask_app.py\",\n        follow_redirects=False\n    )\n    assert res.status_code in [404, 403], f\"Expected 404/403, got {res.status_code}\"\n    assert b\"def static_content\" not in res.data\n\n    # Test 3: Multiple dots traversal\n    res = client.get(\n        \"/static/..../flask_app.py\",\n        follow_redirects=False\n    )\n    assert res.status_code in [404, 403], f\"Expected 404/403, got {res.status_code}\"\n    assert b\"def static_content\" not in res.data\n\n    # Test 4: Try to access other application files\n    for filename in [\"__init__.py\", \"datastore.py\", \"store.py\"]:\n        res = client.get(\n            f\"/static/%2e%2e/{filename}\",\n            follow_redirects=False\n        )\n        assert res.status_code in [404, 403], f\"File {filename} should be blocked\"\n        # Should not contain Python code indicators\n        assert b\"import\" not in res.data or b\"# Test\" in res.data  # Allow \"1 Imported\" etc\n\n    # Test 5: Verify legitimate static files still work\n    # Note: We can't test actual files without knowing what exists,\n    # but we can verify the sanitization doesn't break valid groups\n    res = client.get(\n        \"/static/images/test.png\",  # Will 404 if file doesn't exist, but won't traverse\n        follow_redirects=False\n    )\n    # Should get 404 (file not found) not 403 (blocked)\n    # This confirms the group name \"images\" is valid\n    assert res.status_code == 404\n\n    # Test 6: Ensure hyphens and dots are blocked in group names\n    res = client.get(\n        \"/static/../../../etc/passwd\",\n        follow_redirects=False\n    )\n    assert res.status_code in [404, 403]\n    assert b\"root:\" not in res.data\n\n    # Test 7: Test that underscores still work (they're allowed)\n    res = client.get(\n        \"/static/visual_selector_data/test.json\",\n        follow_redirects=False\n    )\n    # visual_selector_data is a real group, but requires auth\n    # Should get 403 (not authenticated) or 404 (file not found), not a path traversal\n    assert res.status_code in [403, 404]\n\n\ndef test_ssrf_private_ip_blocked(client, live_server, monkeypatch, measure_memory_usage, datastore_path):\n    \"\"\"\n    SSRF protection: IANA-reserved/private IP addresses are blocked at fetch-time, not add-time.\n\n    Watches targeting private/reserved IPs can be *added* freely; the block happens when the\n    fetcher actually tries to reach the URL (via validate_iana_url() in call_browser()).\n\n    Covers:\n    1. is_private_hostname() correctly classifies all reserved ranges\n    2. is_safe_valid_url() ALLOWS private-IP URLs at add-time (IANA check moved to fetch-time)\n    3. ALLOW_IANA_RESTRICTED_ADDRESSES has no effect on add-time; it only controls fetch-time\n    4. UI form accepts private-IP URLs at add-time without error\n    5. Requests fetcher blocks fetch-time DNS rebinding (fresh check on every fetch)\n    6. Requests fetcher blocks redirects that lead to a private IP (open-redirect bypass)\n\n    conftest.py sets ALLOW_IANA_RESTRICTED_ADDRESSES=true globally so the test\n    server (localhost) keeps working for all other tests.  monkeypatch temporarily\n    overrides it to 'false' here, and is automatically restored after the test.\n    \"\"\"\n    from unittest.mock import patch, MagicMock\n    from changedetectionio.validate_url import is_safe_valid_url, is_private_hostname\n\n    monkeypatch.setenv('ALLOW_IANA_RESTRICTED_ADDRESSES', 'false')\n\n    # ------------------------------------------------------------------\n    # 1. is_private_hostname() — unit tests across all reserved ranges\n    # ------------------------------------------------------------------\n    private_hosts = [\n        '127.0.0.1',          # loopback\n        '10.0.0.1',           # RFC 1918\n        '172.16.0.1',         # RFC 1918\n        '192.168.1.1',        # RFC 1918\n        '169.254.169.254',    # link-local / AWS metadata endpoint\n        '::1',                # IPv6 loopback\n        'fc00::1',            # IPv6 unique local\n        'fe80::1',            # IPv6 link-local\n    ]\n    for host in private_hosts:\n        assert is_private_hostname(host), f\"{host} should be identified as private/reserved\"\n\n    for host in ['8.8.8.8', '1.1.1.1']:\n        assert not is_private_hostname(host), f\"{host} should be identified as public\"\n\n    # ------------------------------------------------------------------\n    # 2. is_safe_valid_url() ALLOWS private-IP URLs at add-time\n    #    IANA check is no longer done here — it moved to fetch-time validate_iana_url()\n    # ------------------------------------------------------------------\n    private_ip_urls = [\n        'http://127.0.0.1/',\n        'http://10.0.0.1/',\n        'http://172.16.0.1/',\n        'http://192.168.1.1/',\n        'http://169.254.169.254/',\n        'http://169.254.169.254/latest/meta-data/iam/security-credentials/',\n        'http://[::1]/',\n        'http://[fc00::1]/',\n        'http://[fe80::1]/',\n    ]\n    for url in private_ip_urls:\n        assert is_safe_valid_url(url), f\"{url} should be allowed by is_safe_valid_url (IANA check is at fetch-time)\"\n\n    # ------------------------------------------------------------------\n    # 3. ALLOW_IANA_RESTRICTED_ADDRESSES does not affect add-time validation\n    #    It only controls fetch-time blocking inside validate_iana_url()\n    # ------------------------------------------------------------------\n    monkeypatch.setenv('ALLOW_IANA_RESTRICTED_ADDRESSES', 'true')\n    assert is_safe_valid_url('http://127.0.0.1/'), \\\n        \"Private IP should be allowed at add-time regardless of ALLOW_IANA_RESTRICTED_ADDRESSES\"\n\n    monkeypatch.setenv('ALLOW_IANA_RESTRICTED_ADDRESSES', 'false')\n    assert is_safe_valid_url('http://127.0.0.1/'), \\\n        \"Private IP should be allowed at add-time regardless of ALLOW_IANA_RESTRICTED_ADDRESSES\"\n\n    # ------------------------------------------------------------------\n    # 4. UI form accepts private-IP URLs at add-time\n    #    The watch is created; the SSRF block fires later at fetch-time\n    # ------------------------------------------------------------------\n    for url in ['http://127.0.0.1/', 'http://169.254.169.254/latest/meta-data/']:\n        res = client.post(\n            url_for('ui.ui_views.form_quick_watch_add'),\n            data={'url': url, 'tags': ''},\n            follow_redirects=True\n        )\n        assert b'Watch protocol is not permitted or invalid URL format' not in res.data, \\\n            f\"UI should accept {url} at add-time (SSRF is blocked at fetch-time)\"\n\n    # ------------------------------------------------------------------\n    # 5. Fetch-time DNS-rebinding check in the requests fetcher\n    #    Simulates: URL passed add-time validation with a public IP, but\n    #    by fetch time DNS has been rebound to a private IP.\n    # ------------------------------------------------------------------\n    from changedetectionio.content_fetchers.requests import fetcher as RequestsFetcher\n\n    f = RequestsFetcher()\n\n    with patch('changedetectionio.content_fetchers.requests.is_private_hostname', return_value=True):\n        with pytest.raises(Exception, match='private/reserved'):\n            f._run_sync(\n                url='http://example.com/',\n                timeout=5,\n                request_headers={},\n                request_body=None,\n                request_method='GET',\n            )\n\n    # ------------------------------------------------------------------\n    # 6. Redirect-to-private-IP blocked (open-redirect SSRF bypass)\n    #    Public host returns a 302 pointing at an IANA-reserved address.\n    # ------------------------------------------------------------------\n    mock_redirect = MagicMock()\n    mock_redirect.is_redirect = True\n    mock_redirect.status_code = 302\n    mock_redirect.headers = {'Location': 'http://169.254.169.254/latest/meta-data/'}\n\n    def _private_only_for_redirect(hostname):\n        # Initial host is \"public\"; the redirect target is private\n        return hostname in {'169.254.169.254', '10.0.0.1', '172.16.0.1',\n                            '192.168.0.1', '127.0.0.1', '::1'}\n\n    with patch('changedetectionio.content_fetchers.requests.is_private_hostname',\n               side_effect=_private_only_for_redirect):\n        with patch('requests.Session.request', return_value=mock_redirect):\n            with pytest.raises(Exception, match='Redirect blocked'):\n                f._run_sync(\n                    url='http://example.com/',\n                    timeout=5,\n                    request_headers={},\n                    request_body=None,\n                    request_method='GET',\n                )\n\n\ndef test_unresolvable_hostname_is_allowed(client, live_server, monkeypatch):\n    \"\"\"\n    Unresolvable hostnames must NOT be blocked at add-time when ALLOW_IANA_RESTRICTED_ADDRESSES=false.\n\n    DNS failure (gaierror) at add-time does not mean the URL resolves to a private IP —\n    the domain may simply be offline or not yet live. Blocking it would be a false positive.\n    The real DNS-rebinding protection happens at fetch-time in call_browser().\n    \"\"\"\n    from changedetectionio.validate_url import is_safe_valid_url\n\n    monkeypatch.setenv('ALLOW_IANA_RESTRICTED_ADDRESSES', 'false')\n\n    url = 'http://this-host-does-not-exist-xyz987.invalid/some/path'\n\n    # Should pass URL validation despite being unresolvable\n    assert is_safe_valid_url(url), \\\n        \"Unresolvable hostname should pass is_safe_valid_url — DNS failure is not a private-IP signal\"\n\n    # Should be accepted via the UI form and appear in the watch list\n    res = client.post(\n        url_for('ui.ui_views.form_quick_watch_add'),\n        data={'url': url, 'tags': ''},\n        follow_redirects=True\n    )\n    assert b'Watch protocol is not permitted or invalid URL format' not in res.data, \\\n        \"UI should not reject a URL just because its hostname is unresolvable\"\n\n    res = client.get(url_for('watchlist.index'))\n    assert b'this-host-does-not-exist-xyz987.invalid' in res.data, \\\n        \"Unresolvable hostname watch should appear in the watch overview list\"\n"
  },
  {
    "path": "changedetectionio/tests/test_settings_tag_force_reprocess.py",
    "content": "#!/usr/bin/env python3\n\"\"\"\nTest that changing global settings or tag configurations forces reprocessing.\n\nWhen settings or tag configurations change, all affected watches need to\nreprocess even if their content hasn't changed, because configuration affects\nthe processing result.\n\"\"\"\n\nimport os\nimport time\nfrom flask import url_for\nfrom .util import wait_for_all_checks\n\n\ndef test_settings_change_forces_reprocess(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test that changing global settings clears all checksums to force reprocessing.\n    \"\"\"\n\n    # Setup test content\n    test_html = \"\"\"\n     \n     
    Test content that stays the same
    \n     \n     \n    \"\"\"\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_html)\n\n    test_url = url_for('test_endpoint', _external=True)\n\n    # Add two watches\n    datastore = client.application.config.get('DATASTORE')\n    uuid1 = datastore.add_watch(url=test_url, extras={'title': 'Watch 1'})\n    uuid2 = datastore.add_watch(url=test_url, extras={'title': 'Watch 2'})\n\n    # Unpause watches\n    datastore.data['watching'][uuid1]['paused'] = False\n    datastore.data['watching'][uuid2]['paused'] = False\n\n    # First check - establishes baseline\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    # Verify checksum files were created\n    checksum1 = os.path.join(datastore_path, uuid1, 'last-checksum.txt')\n    checksum2 = os.path.join(datastore_path, uuid2, 'last-checksum.txt')\n    assert os.path.isfile(checksum1), \"First check should create checksum file for watch 1\"\n    assert os.path.isfile(checksum2), \"First check should create checksum file for watch 2\"\n\n    # Change global settings (any setting will do)\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\n            \"application-empty_pages_are_a_change\": \"\",\n            \"requests-time_between_check-minutes\": 180,\n            'application-fetch_backend': \"html_requests\"\n        },\n        follow_redirects=True\n    )\n    assert b\"Settings updated.\" in res.data\n\n    # Give it a moment to process\n    time.sleep(0.5)\n\n    # Verify ALL checksum files were deleted\n    assert not os.path.isfile(checksum1), \"Settings change should delete checksum for watch 1\"\n    assert not os.path.isfile(checksum2), \"Settings change should delete checksum for watch 2\"\n\n    # Next check should reprocess (not skip) and recreate checksums\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    # Verify checksum files were recreated\n    assert os.path.isfile(checksum1), \"Reprocessing should recreate checksum file for watch 1\"\n    assert os.path.isfile(checksum2), \"Reprocessing should recreate checksum file for watch 2\"\n\n    print(\"✓ Settings change forces reprocessing of all watches\")\n\n\ndef test_tag_change_forces_reprocess(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test that changing a tag configuration clears checksums only for watches with that tag.\n    \"\"\"\n\n    # Setup test content\n    test_html = \"\"\"\n     \n     
    Test content that stays the same
    \n     \n     \n    \"\"\"\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_html)\n\n    test_url = url_for('test_endpoint', _external=True)\n\n    # Create a tag\n    datastore = client.application.config.get('DATASTORE')\n    tag_uuid = datastore.add_tag('Test Tag')\n\n    # Add watches - one with tag, one without\n    uuid_with_tag = datastore.add_watch(url=test_url, extras={'title': 'Watch With Tag', 'tags': [tag_uuid]})\n    uuid_without_tag = datastore.add_watch(url=test_url, extras={'title': 'Watch Without Tag'})\n\n    # Unpause watches\n    datastore.data['watching'][uuid_with_tag]['paused'] = False\n    datastore.data['watching'][uuid_without_tag]['paused'] = False\n\n    # First check - establishes baseline\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    # Verify checksum files were created\n    checksum_with = os.path.join(datastore_path, uuid_with_tag, 'last-checksum.txt')\n    checksum_without = os.path.join(datastore_path, uuid_without_tag, 'last-checksum.txt')\n    assert os.path.isfile(checksum_with), \"First check should create checksum for tagged watch\"\n    assert os.path.isfile(checksum_without), \"First check should create checksum for untagged watch\"\n\n    # Edit the tag (change notification_muted as an example)\n    tag = datastore.data['settings']['application']['tags'][tag_uuid]\n    res = client.post(\n        url_for(\"tags.form_tag_edit_submit\", uuid=tag_uuid),\n        data={\n            'title': 'Test Tag',\n            'notification_muted': 'y',\n            'overrides_watch': 'n'\n        },\n        follow_redirects=True\n    )\n    assert b\"Updated\" in res.data\n\n    # Give it a moment to process\n    time.sleep(0.5)\n\n    # Verify ONLY the tagged watch's checksum was deleted\n    assert not os.path.isfile(checksum_with), \"Tag change should delete checksum for watch WITH tag\"\n    assert os.path.isfile(checksum_without), \"Tag change should NOT delete checksum for watch WITHOUT tag\"\n\n    # Next check should reprocess tagged watch and recreate its checksum\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    # Verify tagged watch's checksum was recreated\n    assert os.path.isfile(checksum_with), \"Reprocessing should recreate checksum for tagged watch\"\n    assert os.path.isfile(checksum_without), \"Untagged watch should still have its checksum\"\n\n    print(\"✓ Tag change forces reprocessing only for watches with that tag\")\n\n\ndef test_tag_change_via_api_forces_reprocess(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test that updating a tag via API also clears checksums for affected watches.\n    \"\"\"\n\n    # Setup test content\n    test_html = \"\"\"\n     \n     
    Test content
    \n     \n     \n    \"\"\"\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_html)\n\n    test_url = url_for('test_endpoint', _external=True)\n\n    # Create a tag\n    datastore = client.application.config.get('DATASTORE')\n    tag_uuid = datastore.add_tag('API Test Tag')\n\n    # Add watch with tag\n    uuid_with_tag = datastore.add_watch(url=test_url, extras={'title': 'API Watch'})\n    datastore.data['watching'][uuid_with_tag]['paused'] = False\n    datastore.data['watching'][uuid_with_tag]['tags'] = [tag_uuid]\n    datastore.data['watching'][uuid_with_tag].commit()\n\n    # First check\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    # Verify checksum exists\n    checksum_file = os.path.join(datastore_path, uuid_with_tag, 'last-checksum.txt')\n    assert os.path.isfile(checksum_file), \"First check should create checksum file\"\n\n    # Update tag via API\n    res = client.put(\n        f'/api/v1/tag/{tag_uuid}',\n        json={'notification_muted': True},\n        headers={'x-api-key': datastore.data['settings']['application']['api_access_token']}\n    )\n    assert res.status_code == 200, f\"API call failed with status {res.status_code}: {res.data}\"\n\n    # Give it more time for async operations\n    time.sleep(1.0)\n\n    # Debug: Check if checksum still exists\n    if os.path.isfile(checksum_file):\n        # Read checksum to see if it changed\n        with open(checksum_file, 'r') as f:\n            checksum_content = f.read()\n            print(f\"Checksum still exists: {checksum_content}\")\n\n    # Verify checksum was deleted\n    assert not os.path.isfile(checksum_file), \"API tag update should delete checksum\"\n\n    print(\"✓ Tag update via API forces reprocessing\")\n"
  },
  {
    "path": "changedetectionio/tests/test_share_watch.py",
    "content": "#!/usr/bin/env python3\n\nimport time\nfrom flask import url_for\nfrom urllib.request import urlopen\nfrom .util import set_original_response, set_modified_response, live_server_setup, delete_all_watches\nimport re\n\ndef test_share_watch(client, live_server, measure_memory_usage, datastore_path):\n    set_original_response(datastore_path=datastore_path)\n\n   #  live_server_setup(live_server) # Setup on conftest per function\n\n    test_url = url_for('test_endpoint', _external=True)\n    include_filters = \".nice-filter\"\n\n    # Add our URL to the import page\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    # Goto the edit page, add our ignore text\n    # Add our URL to the import page\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=uuid),\n        data={\"include_filters\": include_filters, \"url\": test_url, \"tags\": \"\", \"headers\": \"\", 'fetch_backend': \"html_requests\", \"time_between_check_use_default\": \"y\"},\n        follow_redirects=True\n    )\n    assert b\"Updated watch.\" in res.data\n    # Check it saved\n    res = client.get(\n        url_for(\"ui.ui_edit.edit_page\", uuid=uuid),\n    )\n    assert bytes(include_filters.encode('utf-8')) in res.data\n\n    # click share the link\n    res = client.get(\n        url_for(\"ui.form_share_put_watch\", uuid=uuid),\n        follow_redirects=True\n    )\n\n    assert b\"Share this link:\" in res.data\n    assert b\"https://changedetection.io/share/\" in res.data\n\n    html = res.data.decode()\n    share_link_search = re.search('(.*)', html, re.IGNORECASE)\n    assert share_link_search\n\n    # Now delete what we have, we will try to re-import it\n    # Cleanup everything\n    delete_all_watches(client)\n\n    # Add our URL to the import page\n    res = client.post(\n        url_for(\"imports.import_page\"),\n        data={\"urls\": share_link_search.group(1)},\n        follow_redirects=True\n    )\n\n    assert b\"1 Imported\" in res.data\n\n    # Now hit edit, we should see what we expect\n    # that the import fetched the meta-data\n    uuids = list(client.application.config.get('DATASTORE').data['watching'])\n    assert uuids, \"It saved/imported and created a new URL from the share\"\n    # Check it saved\n    res = client.get(\n        url_for(\"ui.ui_edit.edit_page\", uuid=uuids[0]),\n    )\n    assert bytes(include_filters.encode('utf-8')) in res.data\n\n    # Check it saved the URL\n    res = client.get(url_for(\"watchlist.index\"))\n    assert bytes(test_url.encode('utf-8')) in res.data\n\n    delete_all_watches(client)"
  },
  {
    "path": "changedetectionio/tests/test_source.py",
    "content": "#!/usr/bin/env python3\n\nimport time\nfrom flask import url_for\nfrom urllib.request import urlopen\nfrom .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks\nfrom ..diff import ADDED_STYLE\n\n\ndef test_check_basic_change_detection_functionality_source(client, live_server, measure_memory_usage, datastore_path):\n    set_original_response(datastore_path=datastore_path)\n\n    test_url = 'source:'+url_for('test_endpoint', _external=True)\n    # Add our URL to the import page\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    wait_for_all_checks(client)\n\n    #####################\n\n    # Check HTML conversion detected and workd\n    res = client.get(\n        url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"),\n        follow_redirects=True\n    )\n\n    # Check this class DOES appear (that we didnt see the actual source)\n    assert b'foobar-detection' in res.data\n\n    # Make a change\n    set_modified_response(datastore_path=datastore_path)\n\n    # Force recheck\n    res = client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    assert b'Queued 1 watch for rechecking.' in res.data\n\n    wait_for_all_checks(client)\n\n    # Now something should be ready, indicated by having a 'has-unread-changes' class\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'has-unread-changes' in res.data\n\n    res = client.get(\n        url_for(\"ui.ui_diff.diff_history_page\", uuid=\"first\"),\n        follow_redirects=True\n    )\n    # With diff-match-patch, HTML tags are properly tokenized and excluded from diff spans\n    # Only \"modified\" is shown as added, while  and  tags remain unchanged\n    assert b'aria-label=\"Changed into\" title=\"Changed into\">' in res.data\n    assert b'<title>modified head title'\n\n# `subtractive_selectors` should still work in `source:` type requests\ndef test_check_ignore_elements(client, live_server, measure_memory_usage, datastore_path):\n    set_original_response(datastore_path=datastore_path)\n\n    time.sleep(1)\n    test_url = 'source:'+url_for('test_endpoint', _external=True)\n    # Add our URL to the import page\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    wait_for_all_checks(client)\n\n    #####################\n    # We want <span> and <p> ONLY, but ignore span with .foobar-detection\n\n    client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={\"include_filters\": 'span,p', \"url\": test_url, \"tags\": \"\", \"subtractive_selectors\": \".foobar-detection\", 'fetch_backend': \"html_requests\", \"time_between_check_use_default\": \"y\"},\n        follow_redirects=True\n    )\n\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    wait_for_all_checks(client)\n\n\n    res = client.get(\n        url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"),\n        follow_redirects=True\n    )\n    assert b'foobar-detection' not in res.data\n    assert b'<br' not in res.data\n    assert b'<p' in res.data\n"
  },
  {
    "path": "changedetectionio/tests/test_trigger.py",
    "content": "#!/usr/bin/env python3\n\nimport time\nfrom flask import url_for\nfrom .util import live_server_setup, wait_for_all_checks\nimport os\n\n\ndef set_original_ignore_response(datastore_path):\n    test_return_data = \"\"\"<html>\n       <body>\n     Some initial text<br>\n     <p>Which is across multiple lines</p>\n     <br>\n     So let's see what happens.  <br>\n     and more<br>\n     </body>\n     </html>\n\n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n\n\ndef set_modified_original_ignore_response(datastore_path):\n    test_return_data = \"\"\"<html>\n       <body>\n     Some NEW nice initial text<br>\n     <p>Which is across multiple lines</p>\n     <br>\n     So let's see what happens.  <br>\n     and more<br>\n     </body>\n     </html>\n\n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n\n\ndef set_modified_with_trigger_text_response(datastore_path):\n    test_return_data = \"\"\"<html>\n       <body>\n     Some NEW nice initial text<br>\n     <p>Which is across multiple lines</p>\n     <br>\n     Add to cart\n     <br>\n     So let's see what happens.  <br>\n     and more<br>\n     </body>\n     </html>\n\n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n\n\ndef test_trigger_functionality(client, live_server, measure_memory_usage, datastore_path):\n\n   #  live_server_setup(live_server) # Setup on conftest per function\n    trigger_text = \"Add to cart\"\n    set_original_ignore_response(datastore_path=datastore_path)\n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    # Trigger a check\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    # Goto the edit page, add our ignore text\n    # Add our URL to the import page\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={\"trigger_text\": trigger_text,\n              \"ignore_text\": \"and more\",\n              \"url\": test_url,\n              \"fetch_backend\": \"html_requests\",\n              \"time_between_check_use_default\": \"y\"},\n        follow_redirects=True\n    )\n    assert b\"Updated watch.\" in res.data\n\n    wait_for_all_checks(client)\n    # Check it saved\n    res = client.get(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n    )\n    assert bytes(trigger_text.encode('utf-8')) in res.data\n\n\n    \n    # so that we set the state to 'has-unread-changes' after all the edits\n    client.get(url_for(\"ui.ui_diff.diff_history_page\", uuid=\"first\"))\n\n    # Trigger a check\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    wait_for_all_checks(client)\n\n    # It should report nothing found (no new 'has-unread-changes' class)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'has-unread-changes' not in res.data\n    assert b'/test-endpoint' in res.data\n\n    #  Make a change\n    set_modified_original_ignore_response(datastore_path=datastore_path)\n\n    # Trigger a check\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    # It should report nothing found (no new 'has-unread-changes' class)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'has-unread-changes' not in res.data\n\n    # Now set the content which contains the trigger text\n    set_modified_with_trigger_text_response(datastore_path=datastore_path)\n\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'has-unread-changes' in res.data\n\n    # https://github.com/dgtlmoon/changedetection.io/issues/616\n    # Apparently the actual snapshot that contains the trigger never shows\n    res = client.get(url_for(\"ui.ui_diff.diff_history_page\", uuid=\"first\"))\n    assert b'Add to cart' in res.data\n\n    # Check the preview/highlighter, we should be able to see what we triggered on, but it should be highlighted\n    res = client.get(url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"))\n    assert b'ignored_line_numbers = [8]' in res.data\n\n    # We should be able to see what we triggered on\n    # The JS highlighter should tell us which lines (also used in the live-preview)\n    assert b'const triggered_line_numbers = [6]' in res.data\n    assert b'Add to cart' in res.data\n\n"
  },
  {
    "path": "changedetectionio/tests/test_trigger_regex.py",
    "content": "#!/usr/bin/env python3\n\nimport time\nfrom flask import url_for\nfrom .util import live_server_setup, wait_for_all_checks, delete_all_watches\nimport os\n\n\ndef set_original_ignore_response(datastore_path):\n    test_return_data = \"\"\"<html>\n       <body>\n     Some initial text<br>\n     <p>Which is across multiple lines</p>\n     <br>\n     So let's see what happens.  <br>\n     </body>\n     </html>\n\n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n\n\n\ndef test_trigger_regex_functionality(client, live_server, measure_memory_usage, datastore_path):\n\n   #  live_server_setup(live_server) # Setup on conftest per function\n\n    set_original_ignore_response(datastore_path=datastore_path)\n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    # Give the thread time to pick it up\n    wait_for_all_checks(client)\n\n    # It should report nothing found (just a new one shouldnt have anything)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'has-unread-changes' not in res.data\n\n    ### test regex\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={\"trigger_text\": '/something \\d{3}/',\n              \"url\": test_url,\n              \"fetch_backend\": \"html_requests\",\n              \"time_between_check_use_default\": \"y\"},\n        follow_redirects=True\n    )\n    wait_for_all_checks(client)\n    # so that we set the state to 'has-unread-changes' after all the edits\n    client.get(url_for(\"ui.ui_diff.diff_history_page\", uuid=\"first\"))\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(\"some new noise\")\n\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    # It should report nothing found (nothing should match the regex)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'has-unread-changes' not in res.data\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(\"regex test123<br>\\nsomething 123\")\n\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'has-unread-changes' in res.data\n\n    # Cleanup everything\n    delete_all_watches(client)\n"
  },
  {
    "path": "changedetectionio/tests/test_trigger_regex_with_filter.py",
    "content": "#!/usr/bin/env python3\n\nimport time\nfrom flask import url_for\n\nfrom .util import live_server_setup, delete_all_watches, wait_for_all_checks\nimport os\n\n\ndef set_original_ignore_response(datastore_path):\n    test_return_data = \"\"\"<html>\n       <body>\n     Some initial text<br>\n     <p>Which is across multiple lines</p>\n     <br>\n     So let's see what happens.  <br>\n     </body>\n     </html>\n\n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n\n\n\ndef test_trigger_regex_functionality_with_filter(client, live_server, measure_memory_usage, datastore_path):\n\n    set_original_ignore_response(datastore_path=datastore_path)\n\n    # Give the endpoint time to spin up\n    time.sleep(1)\n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    wait_for_all_checks(client)\n\n    ### test regex with filter\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={\"trigger_text\": \"/cool.stuff/\",\n              \"url\": test_url,\n              \"include_filters\": '#in-here',\n              \"fetch_backend\": \"html_requests\",\n              \"time_between_check_use_default\": \"y\"},\n        follow_redirects=True\n    )\n\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    wait_for_all_checks(client)\n\n    client.get(url_for(\"ui.ui_diff.diff_history_page\", uuid=\"first\"))\n\n    # Check that we have the expected text.. but it's not in the css filter we want\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(\"<html>some new noise with cool stuff2 ok</html>\")\n\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    wait_for_all_checks(client)\n\n    # It should report nothing found (nothing should match the regex and filter)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'has-unread-changes' not in res.data\n\n    # now this should trigger something\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(\"<html>some new noise with <span id=in-here>cool stuff6</span> ok</html>\")\n\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'has-unread-changes' in res.data\n\n# Cleanup everything\n    delete_all_watches(client)\n"
  },
  {
    "path": "changedetectionio/tests/test_ui.py",
    "content": "#!/usr/bin/env python3\n\nfrom flask import url_for\nfrom .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks, delete_all_watches\nfrom ..forms import REQUIRE_ATLEAST_ONE_TIME_PART_WHEN_NOT_GLOBAL_DEFAULT, REQUIRE_ATLEAST_ONE_TIME_PART_MESSAGE_DEFAULT\n\n\ndef test_recheck_time_field_validation_global_settings(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Tests that the global settings time field has atleast one value for week/day/hours/minute/seconds etc entered\n    class globalSettingsRequestForm(Form):\n        time_between_check = RequiredFormField(TimeBetweenCheckForm)\n    \"\"\"\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\n              \"requests-time_between_check-weeks\": '',\n              \"requests-time_between_check-days\": '',\n              \"requests-time_between_check-hours\": '',\n              \"requests-time_between_check-minutes\": '',\n              \"requests-time_between_check-seconds\": '',\n              },\n        follow_redirects=True\n    )\n\n\n    assert REQUIRE_ATLEAST_ONE_TIME_PART_MESSAGE_DEFAULT.encode('utf-8') in res.data\n    delete_all_watches(client)\n\n\ndef test_recheck_time_field_validation_single_watch(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Tests that the global settings time field has atleast one value for week/day/hours/minute/seconds etc entered\n    class globalSettingsRequestForm(Form):\n        time_between_check = RequiredFormField(TimeBetweenCheckForm)\n    \"\"\"\n    test_url = url_for('test_endpoint', _external=True)\n\n    # Add our URL to the import page\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={\n            \"url\": test_url,\n            'fetch_backend': \"html_requests\",\n            \"time_between_check_use_default\": \"\",  # OFF\n            \"time_between_check-weeks\": '',\n            \"time_between_check-days\": '',\n            \"time_between_check-hours\": '',\n            \"time_between_check-minutes\": '',\n            \"time_between_check-seconds\": '',\n        },\n        follow_redirects=True\n    )\n\n\n    assert REQUIRE_ATLEAST_ONE_TIME_PART_WHEN_NOT_GLOBAL_DEFAULT.encode('utf-8') in res.data\n\n    # Now set some time\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={\n            \"url\": test_url,\n            'fetch_backend': \"html_requests\",\n            \"time_between_check_use_default\": \"\",  # OFF\n            \"time_between_check-weeks\": '',\n            \"time_between_check-days\": '',\n            \"time_between_check-hours\": '',\n            \"time_between_check-minutes\": '5',\n            \"time_between_check-seconds\": '',\n        },\n        follow_redirects=True\n    )\n\n    assert b\"Updated watch.\" in res.data\n    assert REQUIRE_ATLEAST_ONE_TIME_PART_WHEN_NOT_GLOBAL_DEFAULT.encode('utf-8') not in res.data\n\n    # Now set to use defaults\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={\n            \"url\": test_url,\n            'fetch_backend': \"html_requests\",\n            \"time_between_check_use_default\": \"y\",  # ON YES\n            \"time_between_check-weeks\": '',\n            \"time_between_check-days\": '',\n            \"time_between_check-hours\": '',\n            \"time_between_check-minutes\": '',\n            \"time_between_check-seconds\": '',\n        },\n        follow_redirects=True\n    )\n\n    assert b\"Updated watch.\" in res.data\n    assert REQUIRE_ATLEAST_ONE_TIME_PART_WHEN_NOT_GLOBAL_DEFAULT.encode('utf-8') not in res.data\n    delete_all_watches(client)\n\ndef test_checkbox_open_diff_in_new_tab(client, live_server, measure_memory_usage, datastore_path):\n    \n    set_original_response(datastore_path=datastore_path)\n\n    # Add our URL to the import page\n    res = client.post(\n        url_for(\"imports.import_page\"),\n        data={\"urls\": url_for('test_endpoint', _external=True)},\n        follow_redirects=True\n    )\n\n    assert b\"1 Imported\" in res.data\n    wait_for_all_checks(client)\n\n    # Make a change\n    set_modified_response(datastore_path=datastore_path)\n\n    # Test case 1 - checkbox is enabled in settings\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\"application-ui-open_diff_in_new_tab\": \"1\"},\n        follow_redirects=True\n    )\n    assert b'Settings updated' in res.data\n\n    # Force recheck\n    res = client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    assert b'Queued 1 watch for rechecking.' in res.data\n\n    wait_for_all_checks(client)\n    \n    res = client.get(url_for(\"watchlist.index\"))\n    lines = res.data.decode().split(\"\\n\")\n\n    # Find link to diff page\n    target_line = None\n    for line in lines:\n        if '/diff' in line:\n            target_line = line.strip()\n            break\n\n    assert target_line != None\n    assert 'target=' in target_line\n\n    # Test case 2 - checkbox is disabled in settings\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\"application-ui-open_diff_in_new_tab\": \"\"},\n        follow_redirects=True\n    )\n    assert b'Settings updated' in res.data\n\n    # Force recheck\n    res = client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    assert b'Queued 1 watch for rechecking.' in res.data\n\n    wait_for_all_checks(client)\n    \n    res = client.get(url_for(\"watchlist.index\"))\n    lines = res.data.decode().split(\"\\n\")\n\n    # Find link to diff page\n    target_line = None\n    for line in lines:\n        if '/diff' in line:\n            target_line = line.strip()\n            break\n\n    assert target_line != None\n    assert 'target=' not in target_line\n\n    # Cleanup everything\n    delete_all_watches(client)\n\ndef test_page_title_listing_behaviour(client, live_server, measure_memory_usage, datastore_path):\n\n    set_original_response(extra_title=\"custom html\", datastore_path=datastore_path)\n\n    # either the manually entered title/description or the page link should be visible\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\"application-ui-use_page_title_in_list\": \"\",\n              \"requests-time_between_check-minutes\": 180,\n              'application-fetch_backend': \"html_requests\"},\n        follow_redirects=True\n    )\n    assert b\"Settings updated.\" in res.data\n\n\n    # Add our URL to the import page\n    res = client.post(\n        url_for(\"imports.import_page\"),\n        data={\"urls\": url_for('test_endpoint', _external=True)},\n        follow_redirects=True\n    )\n\n    assert b\"1 Imported\" in res.data\n    wait_for_all_checks(client)\n\n    # We see the URL only, no title/description was manually entered\n    res = client.get(url_for(\"watchlist.index\"))\n    assert url_for('test_endpoint', _external=True).encode('utf-8') in res.data\n\n\n    # Now 'my title' should override\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={\n        \"url\": url_for('test_endpoint', _external=True),\n        \"title\": \"my title\",\n        \"fetch_backend\": \"html_requests\",\n        \"time_between_check_use_default\": \"y\"},\n        follow_redirects=True\n    )\n    assert b\"Updated watch.\" in res.data\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b\"my title\" in res.data\n\n    # Now we enable page <title> and unset the override title/description\n    res = client.post(\n        url_for(\"settings.settings_page\"),\n        data={\"application-ui-use_page_title_in_list\": \"y\",\n              \"requests-time_between_check-minutes\": 180,\n              'application-fetch_backend': \"html_requests\"},\n        follow_redirects=True\n    )\n    assert b\"Settings updated.\" in res.data\n\n    # Page title description override should take precedence\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b\"my title\" in res.data\n\n    # Remove page title description override and it should fall back to title\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={\n        \"url\": url_for('test_endpoint', _external=True),\n        \"title\": \"\",\n        \"fetch_backend\": \"html_requests\",\n        \"time_between_check_use_default\": \"y\"},\n        follow_redirects=True\n    )\n    assert b\"Updated watch.\" in res.data\n\n    # No page title description, and 'use_page_title_in_list' is on, it should show the <title>\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b\"head titlecustom html\" in res.data\n    delete_all_watches(client)\n\n\ndef test_ui_viewed_unread_flag(client, live_server, measure_memory_usage, datastore_path):\n\n    import time\n\n    set_original_response(datastore_path=datastore_path, extra_title=\"custom html\")\n\n    # Add our URL to the import page\n    res = client.post(\n        url_for(\"imports.import_page\"),\n        data={\"urls\": url_for('test_endpoint', _external=True)+\"\\r\\n\"+url_for('test_endpoint', _external=True)},\n        follow_redirects=True\n    )\n\n    assert b\"2 Imported\" in res.data\n    wait_for_all_checks(client)\n\n    set_modified_response(datastore_path=datastore_path)\n    res = client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    assert b'Queued 2 watches for rechecking.' in res.data\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'<span id=\"unread-tab-counter\">2</span>' in res.data\n    assert res.data.count(b'data-watch-uuid') == 2\n\n    # one should now be viewed, but two in total still\n    client.get(url_for(\"ui.ui_diff.diff_history_page\", uuid=\"first\"))\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'<span id=\"unread-tab-counter\">1</span>' in res.data\n    assert res.data.count(b'data-watch-uuid') == 2\n\n    # check ?unread=1 works\n    res = client.get(url_for(\"watchlist.index\")+\"?unread=1\")\n    assert res.data.count(b'data-watch-uuid') == 1\n    assert b'<span id=\"unread-tab-counter\">1</span>' in res.data\n\n    # Mark all viewed test again\n    client.get(url_for(\"ui.mark_all_viewed\"), follow_redirects=True)\n    time.sleep(0.2)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'<span id=\"unread-tab-counter\">0</span>' in res.data\n    delete_all_watches(client)\n"
  },
  {
    "path": "changedetectionio/tests/test_unique_lines.py",
    "content": "#!/usr/bin/env python3\n\nimport time\nfrom flask import url_for\nfrom .util import live_server_setup, wait_for_all_checks, delete_all_watches\nimport os\n\n\ndef set_original_ignore_response(datastore_path):\n    test_return_data = \"\"\"<html>\n     <body>\n     <p>Some initial text</p>\n     <p>Which is across multiple lines</p>\n     <p>So let's see what happens.</p>\n     <p>   So let's see what happens.   <br> </p>\n     <p>A - sortable line</p> \n     </body>\n     </html>\n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n\n\n# The same but just re-ordered the text\ndef set_modified_swapped_lines(datastore_path):\n    # Re-ordered and with some whitespacing, should get stripped() too.\n    test_return_data = \"\"\"<html>\n     <body>\n     <p>Some initial text</p>\n     <p>   So let's see what happens.</p>\n     <p> Which is across multiple lines</p>     \n     </body>\n     </html>\n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n\ndef set_modified_swapped_lines_with_extra_text_for_sorting(datastore_path):\n    test_return_data = \"\"\"<html>\n     <body>\n     <p> Which is across multiple lines</p>     \n     <p>Some initial text</p>\n     <p>   So let's see what happens.</p>\n     <p>Z last</p>\n     <p>0 numerical</p>\n     <p>A uppercase</p>\n     <p>a lowercase</p>     \n     </body>\n     </html>\n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n\n\ndef set_modified_with_trigger_text_response(datastore_path):\n    test_return_data = \"\"\"<html>\n     <body>\n     <p>Some initial text</p>\n     <p>So let's see what happens.</p>\n     <p>and a new line!</p>\n     <p>Which is across multiple lines</p>     \n     </body>\n     </html>\n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n\n# def test_setup(client, live_server, measure_memory_usage, datastore_path):\n   #  live_server_setup(live_server) # Setup on conftest per function\n\ndef test_unique_lines_functionality(client, live_server, measure_memory_usage, datastore_path):\n    \n\n\n    set_original_ignore_response(datastore_path=datastore_path)\n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    # Add our URL to the import page\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={\"check_unique_lines\": \"y\",\n              \"url\": test_url,\n              \"fetch_backend\": \"html_requests\",\n              \"time_between_check_use_default\": \"y\"},\n        follow_redirects=True\n    )\n    assert b\"Updated watch.\" in res.data\n    assert b'has-unread-changes' not in res.data\n\n    #  Make a change\n    set_modified_swapped_lines(datastore_path)\n\n    # Trigger a check\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    # Give the thread time to pick it up\n    wait_for_all_checks(client)\n\n    # It should report nothing found (no new 'has-unread-changes' class)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'has-unread-changes' not in res.data\n\n    # Now set the content which contains the new text and re-ordered existing text\n    set_modified_with_trigger_text_response(datastore_path=datastore_path)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'has-unread-changes' in res.data\n    delete_all_watches(client)\n\ndef test_sort_lines_functionality(client, live_server, measure_memory_usage, datastore_path):\n    \n\n    set_modified_swapped_lines_with_extra_text_for_sorting(datastore_path=datastore_path)\n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    # Add our URL to the import page\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={\"sort_text_alphabetically\": \"n\",\n              \"url\": test_url,\n              \"fetch_backend\": \"html_requests\",\n              \"time_between_check_use_default\": \"y\"},\n        follow_redirects=True\n    )\n    assert b\"Updated watch.\" in res.data\n\n\n    # Trigger a check\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    # Give the thread time to pick it up\n    wait_for_all_checks(client)\n\n\n    res = client.get(url_for(\"watchlist.index\"))\n    # Should be a change registered\n    assert b'has-unread-changes' in res.data\n\n    res = client.get(\n        url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"),\n        follow_redirects=True\n    )\n\n    assert res.data.find(b'0 numerical') < res.data.find(b'Z last')\n    assert res.data.find(b'A uppercase') < res.data.find(b'Z last')\n    assert res.data.find(b'Some initial text') < res.data.find(b'Which is across multiple lines')\n    \n    delete_all_watches(client)\n\n\ndef test_extra_filters(client, live_server, measure_memory_usage, datastore_path):\n    \n\n    set_original_ignore_response(datastore_path=datastore_path)\n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    # Add our URL to the import page\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={\"remove_duplicate_lines\": \"y\",\n              \"trim_text_whitespace\": \"y\",\n              \"sort_text_alphabetically\": \"\",  # leave this OFF for testing\n              \"url\": test_url,\n              \"fetch_backend\": \"html_requests\",\n              \"time_between_check_use_default\": \"y\"},\n        follow_redirects=True\n    )\n    assert b\"Updated watch.\" in res.data\n    # Give the thread time to pick it up\n    wait_for_all_checks(client)\n    # Trigger a check\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    # Give the thread time to pick it up\n    wait_for_all_checks(client)\n\n    res = client.get(\n        url_for(\"ui.ui_preview.preview_page\", uuid=\"first\")\n    )\n\n    assert res.data.count(b\"see what happens.\") == 1\n\n    # still should remain unsorted ('A - sortable line') stays at the end\n    assert res.data.find(b'A - sortable line') > res.data.find(b'Which is across multiple lines')\n\n    delete_all_watches(client)"
  },
  {
    "path": "changedetectionio/tests/test_watch_edited_flag.py",
    "content": "#!/usr/bin/env python3\n\"\"\"\nTest the watch edited flag functionality.\n\nThis tests the private __watch_was_edited flag that tracks when writable\nwatch fields are modified, which prevents skipping reprocessing when the\nwatch configuration has changed.\n\"\"\"\n\nimport os\nimport time\nfrom flask import url_for\nfrom .util import live_server_setup, wait_for_all_checks\n\n\ndef set_test_content(datastore_path):\n    \"\"\"Write test HTML content to endpoint-content.txt for test server.\"\"\"\n    test_html = \"\"\"<html>\n     <body>\n     <p>Test content for watch edited flag tests</p>\n     <p>This content stays the same across checks</p>\n     </body>\n     </html>\n    \"\"\"\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_html)\n\n\ndef test_watch_edited_flag_lifecycle(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test the full lifecycle of the was_edited flag:\n    1. Flag starts False when watch is created\n    2. Flag becomes True when writable fields are modified\n    3. Flag is reset False after worker processing\n    4. Flag stays False when readonly fields are modified\n    \"\"\"\n\n    # Setup - Add a watch\n    test_url = url_for('test_endpoint', _external=True)\n    res = client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": test_url, \"tags\": \"\", \"edit_and_watch_submit_button\": \"Edit > Watch\"},\n        follow_redirects=True\n    )\n    assert b\"Watch added\" in res.data or b\"Updated watch\" in res.data\n\n    # Get the watch UUID\n    datastore = client.application.config.get('DATASTORE')\n    uuid = list(datastore.data['watching'].keys())[0]\n    watch = datastore.data['watching'][uuid]\n\n    # Reset flag after initial form submission (form sets fields which trigger the flag)\n    watch.reset_watch_edited_flag()\n\n    # Test 1: Flag should be False after reset\n    assert not watch.was_edited, \"Flag should be False after reset\"\n\n    # Test 2: Modify a writable field (title) - flag should become True\n    watch['title'] = 'New Title'\n    assert watch.was_edited, \"Flag should be True after modifying writable field 'title'\"\n\n    # Test 3: Reset flag manually (simulating what worker does)\n    watch.reset_watch_edited_flag()\n    assert not watch.was_edited, \"Flag should be False after reset\"\n\n    # Test 4: Modify another writable field (url) - flag should become True again\n    watch['url'] = 'https://example.com'\n    assert watch.was_edited, \"Flag should be True after modifying writable field 'url'\"\n\n    # Test 5: Reset and modify a readonly field - flag should stay False\n    watch.reset_watch_edited_flag()\n    assert not watch.was_edited, \"Flag should be False after reset\"\n\n    # Modify readonly field (uuid) - should not set flag\n    old_uuid = watch['uuid']\n    watch['uuid'] = 'readonly-test-uuid'\n    assert not watch.was_edited, \"Flag should stay False when modifying readonly field 'uuid'\"\n    watch['uuid'] = old_uuid  # Restore original\n\n    # Note: Worker reset behavior is tested in test_check_removed_line_contains_trigger\n    # and test_watch_edited_flag_prevents_skip\n\n    print(\"✓ All watch edited flag lifecycle tests passed\")\n\n\ndef test_watch_edited_flag_dict_methods(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test that the flag is set correctly by various dict methods:\n    - __setitem__ (watch['key'] = value)\n    - update() (watch.update({'key': value}))\n    - setdefault() (watch.setdefault('key', default))\n    - pop() (watch.pop('key'))\n    - __delitem__ (del watch['key'])\n    \"\"\"\n\n    # Setup - Add a watch\n    test_url = url_for('test_endpoint', _external=True)\n    res = client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": test_url, \"tags\": \"\", \"edit_and_watch_submit_button\": \"Edit > Watch\"},\n        follow_redirects=True\n    )\n\n    datastore = client.application.config.get('DATASTORE')\n    uuid = list(datastore.data['watching'].keys())[0]\n    watch = datastore.data['watching'][uuid]\n\n    # Test __setitem__\n    watch.reset_watch_edited_flag()\n    watch['title'] = 'Test via setitem'\n    assert watch.was_edited, \"Flag should be True after __setitem__ on writable field\"\n\n    # Test update() with dict\n    watch.reset_watch_edited_flag()\n    watch.update({'title': 'Test via update dict'})\n    assert watch.was_edited, \"Flag should be True after update() with writable field\"\n\n    # Test update() with kwargs\n    watch.reset_watch_edited_flag()\n    watch.update(title='Test via update kwargs')\n    assert watch.was_edited, \"Flag should be True after update() kwargs with writable field\"\n\n    # Test setdefault() on new key\n    watch.reset_watch_edited_flag()\n    watch.setdefault('title', 'Should not be set')  # Key exists, no change\n    assert not watch.was_edited, \"Flag should stay False when setdefault() doesn't change existing key\"\n\n    watch.setdefault('custom_field', 'New value')  # New key\n    assert watch.was_edited, \"Flag should be True after setdefault() creates new writable field\"\n\n    # Test pop() on writable field\n    watch.reset_watch_edited_flag()\n    watch.pop('custom_field', None)\n    assert watch.was_edited, \"Flag should be True after pop() on writable field\"\n\n    # Test __delitem__ on writable field\n    watch.reset_watch_edited_flag()\n    watch['temp_field'] = 'temp'\n    watch.reset_watch_edited_flag()  # Reset after adding\n    del watch['temp_field']\n    assert watch.was_edited, \"Flag should be True after __delitem__ on writable field\"\n\n    print(\"✓ All dict methods correctly set the flag\")\n\n\ndef test_watch_edited_flag_prevents_skip(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test that the was_edited flag prevents skipping reprocessing.\n    When watch configuration is edited, it should reprocess even if content unchanged.\n    After worker processing, flag should be reset and subsequent checks can skip.\n    \"\"\"\n\n    # Setup test content\n    set_test_content(datastore_path)\n\n    # Setup - Add a watch\n    test_url = url_for('test_endpoint', _external=True)\n    res = client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": test_url, \"tags\": \"\", \"edit_and_watch_submit_button\": \"Edit > Watch\"},\n        follow_redirects=True\n    )\n    assert b\"Watch added\" in res.data or b\"Updated watch\" in res.data\n\n    datastore = client.application.config.get('DATASTORE')\n    uuid = list(datastore.data['watching'].keys())[0]\n    watch = datastore.data['watching'][uuid]\n\n    # Unpause the watch (watches are paused by default in tests)\n    watch['paused'] = False\n\n    # Run first check to establish baseline\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    # Verify first check completed successfully - checksum file should exist\n    checksum_file = os.path.join(datastore_path, uuid, 'last-checksum.txt')\n    assert os.path.isfile(checksum_file), \"First check should create last-checksum.txt file\"\n\n    # Reset the was_edited flag (simulating clean state after processing)\n    watch.reset_watch_edited_flag()\n    assert not watch.was_edited, \"Flag should be False after reset\"\n\n    # Run second check without any changes - should skip via checksumFromPreviousCheckWasTheSame\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    # Verify it was skipped (last_check_status should indicate skip)\n    # Note: The actual skip is tested in test_check_removed_line_contains_trigger\n    # Here we're focused on the was_edited flag interaction\n\n    # Now modify the watch - flag should become True\n    watch['title'] = 'Modified Title'\n    assert watch.was_edited, \"Flag should be True after modifying watch\"\n\n    # Run third check - should NOT skip because was_edited=True even though content unchanged\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    # After worker processing, the flag should be reset by the worker\n    # This reset happens in the processor's run() method after processing completes\n    assert not watch.was_edited, \"Flag should be False after worker processing\"\n\n    print(\"✓ was_edited flag correctly prevents skip and is reset by worker\")\n\n\ndef test_watch_edited_flag_system_fields(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"\n    Test that system fields (readonly + additional system fields) don't trigger the flag.\n    \"\"\"\n\n    # Setup - Add a watch\n    test_url = url_for('test_endpoint', _external=True)\n    res = client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": test_url, \"tags\": \"\", \"edit_and_watch_submit_button\": \"Edit > Watch\"},\n        follow_redirects=True\n    )\n\n    datastore = client.application.config.get('DATASTORE')\n    uuid = list(datastore.data['watching'].keys())[0]\n    watch = datastore.data['watching'][uuid]\n\n    # Test readonly fields from OpenAPI spec\n    readonly_fields = ['uuid', 'date_created', 'last_viewed']\n    for field in readonly_fields:\n        watch.reset_watch_edited_flag()\n        if field in watch:\n            old_value = watch[field]\n            watch[field] = 'modified-readonly-value'\n            assert not watch.was_edited, f\"Flag should stay False when modifying readonly field '{field}'\"\n            watch[field] = old_value  # Restore\n\n    # Test additional system fields not in OpenAPI spec yet\n    system_fields = ['last_check_status']\n    for field in system_fields:\n        watch.reset_watch_edited_flag()\n        watch[field] = 'system-value'\n        assert not watch.was_edited, f\"Flag should stay False when modifying system field '{field}'\"\n\n    # Test that content-type (readonly per OpenAPI) doesn't trigger flag\n    watch.reset_watch_edited_flag()\n    watch['content-type'] = 'text/html'\n    assert not watch.was_edited, \"Flag should stay False when modifying 'content-type' (readonly)\"\n\n    print(\"✓ System fields correctly don't trigger the flag\")\n"
  },
  {
    "path": "changedetectionio/tests/test_watch_fields_storage.py",
    "content": "import time\nfrom flask import url_for\nfrom urllib.request import urlopen\nfrom . util import set_original_response, set_modified_response, live_server_setup\n\n\ndef test_check_watch_field_storage(client, live_server, measure_memory_usage, datastore_path):\n    set_original_response(datastore_path=datastore_path)\n\n   #  live_server_setup(live_server) # Setup on conftest per function\n\n    test_url = \"http://somerandomsitewewatch.com\"\n\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={ \"notification_urls\": \"json://127.0.0.1:30000\\r\\njson://128.0.0.1\\r\\n\",\n               \"time_between_check-minutes\": 126,\n               \"include_filters\" : \".fooclass\",\n               \"title\" : \"My title\",\n               \"ignore_text\" : \"ignore this\",\n               \"url\": test_url,\n               \"tags\": \"woohoo\",\n               \"headers\": \"curl:foo\",\n               'fetch_backend': \"html_requests\",\n               \"time_between_check_use_default\": \"y\"\n               },\n        follow_redirects=True\n    )\n    assert b\"Updated watch.\" in res.data\n\n    res = client.get(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        follow_redirects=True\n    )\n    # checks that we dont get an error when using blank lines in the field value\n    assert not b\"json://127.0.0.1\\n\\njson\" in res.data\n    assert not b\"json://127.0.0.1\\r\\n\\njson\" in res.data\n    assert not b\"json://127.0.0.1\\r\\n\\rjson\" in res.data\n\n    assert b\"json://127.0.0.1\" in res.data\n    assert b\"json://128.0.0.1\" in res.data\n\n    assert b\"126\" in res.data\n    assert b\".fooclass\" in res.data\n    assert b\"My title\" in res.data\n    assert b\"ignore this\" in res.data\n    assert b\"http://somerandomsitewewatch.com\" in res.data\n    assert b\"woohoo\" in res.data\n    assert b\"curl: foo\" in res.data\n\n"
  },
  {
    "path": "changedetectionio/tests/test_xpath_default_namespace.py",
    "content": "#!/usr/bin/env python3\n\"\"\"\nUnit tests for XPath default namespace handling in RSS/Atom feeds.\nTests the fix for issue where //title/text() returns empty on feeds with default namespaces.\n\nReal-world test data from https://github.com/microsoft/PowerToys/releases.atom\n\"\"\"\n\nimport sys\nimport os\nimport pytest\n\nsys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))\nimport html_tools\n\n\n# Real-world Atom feed with default namespace from GitHub PowerToys releases\n# This is the actual format that was failing before the fix\natom_feed_with_default_ns = \"\"\"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<feed xmlns=\"http://www.w3.org/2005/Atom\" xmlns:media=\"http://search.yahoo.com/mrss/\" xml:lang=\"en-US\">\n  <id>tag:github.com,2008:https://github.com/microsoft/PowerToys/releases</id>\n  <link type=\"text/html\" rel=\"alternate\" href=\"https://github.com/microsoft/PowerToys/releases\"/>\n  <link type=\"application/atom+xml\" rel=\"self\" href=\"https://github.com/microsoft/PowerToys/releases.atom\"/>\n  <title>Release notes from PowerToys\n  2025-10-23T08:53:12Z\n  \n    tag:github.com,2008:Repository/184456251/v0.95.1\n    2025-10-24T14:20:14Z\n    \n    Release 0.95.1\n    <p>This patch release fixes several important stability issues.</p>\n    \n      Jaylyn-Barbee\n    \n  \n  \n    tag:github.com,2008:Repository/184456251/v0.95.0\n    2025-10-17T12:51:21Z\n    \n    Release v0.95.0\n    <p>New features, stability, optimization improvements.</p>\n    \n      Jaylyn-Barbee\n    \n  \n\"\"\"\n\n# RSS feed without default namespace\nrss_feed_no_default_ns = \"\"\"\n\n  \n    Channel Title\n    Channel Description\n    \n      Item 1 Title\n      Item 1 Description\n    \n    \n      Item 2 Title\n      Item 2 Description\n    \n  \n\"\"\"\n\n# RSS 2.0 feed with namespace prefix (not default)\nrss_feed_with_ns_prefix = \"\"\"\n\n  \n    Channel Title\n    \n    \n      Item Title\n      Author Name\n    \n  \n\"\"\"\n\n\nclass TestXPathDefaultNamespace:\n    \"\"\"Test XPath queries on feeds with and without default namespaces.\"\"\"\n\n    def test_atom_feed_simple_xpath_with_xpath_filter(self):\n        \"\"\"Test that //title/text() works on Atom feed with default namespace using xpath_filter.\"\"\"\n        result = html_tools.xpath_filter('//title/text()', atom_feed_with_default_ns, is_xml=True)\n        assert 'Release notes from PowerToys' in result\n        assert 'Release 0.95.1' in result\n        assert 'Release v0.95.0' in result\n\n    def test_atom_feed_nested_xpath_with_xpath_filter(self):\n        \"\"\"Test nested XPath like //entry/title/text() on Atom feed.\"\"\"\n        result = html_tools.xpath_filter('//entry/title/text()', atom_feed_with_default_ns, is_xml=True)\n        assert 'Release 0.95.1' in result\n        assert 'Release v0.95.0' in result\n        # Should NOT include the feed title\n        assert 'Release notes from PowerToys' not in result\n\n    def test_atom_feed_other_elements_with_xpath_filter(self):\n        \"\"\"Test that other elements like //updated/text() work on Atom feed.\"\"\"\n        result = html_tools.xpath_filter('//updated/text()', atom_feed_with_default_ns, is_xml=True)\n        assert '2025-10-23T08:53:12Z' in result\n        assert '2025-10-24T14:20:14Z' in result\n\n    def test_rss_feed_without_namespace(self):\n        \"\"\"Test that //title/text() works on RSS feed without default namespace.\"\"\"\n        result = html_tools.xpath_filter('//title/text()', rss_feed_no_default_ns, is_xml=True)\n        assert 'Channel Title' in result\n        assert 'Item 1 Title' in result\n        assert 'Item 2 Title' in result\n\n    def test_rss_feed_nested_xpath(self):\n        \"\"\"Test nested XPath on RSS feed without default namespace.\"\"\"\n        result = html_tools.xpath_filter('//item/title/text()', rss_feed_no_default_ns, is_xml=True)\n        assert 'Item 1 Title' in result\n        assert 'Item 2 Title' in result\n        # Should NOT include channel title\n        assert 'Channel Title' not in result\n\n    def test_rss_feed_with_prefixed_namespaces(self):\n        \"\"\"Test that feeds with namespace prefixes (not default) still work.\"\"\"\n        result = html_tools.xpath_filter('//title/text()', rss_feed_with_ns_prefix, is_xml=True)\n        assert 'Channel Title' in result\n        assert 'Item Title' in result\n\n    def test_local_name_workaround_still_works(self):\n        \"\"\"Test that local-name() workaround still works for Atom feeds.\"\"\"\n        result = html_tools.xpath_filter('//*[local-name()=\"title\"]/text()', atom_feed_with_default_ns, is_xml=True)\n        assert 'Release notes from PowerToys' in result\n        assert 'Release 0.95.1' in result\n\n    def test_xpath1_filter_without_default_namespace(self):\n        \"\"\"Test xpath1_filter works on RSS without default namespace.\"\"\"\n        result = html_tools.xpath1_filter('//title/text()', rss_feed_no_default_ns, is_xml=True)\n        assert 'Channel Title' in result\n        assert 'Item 1 Title' in result\n\n    def test_xpath1_filter_with_default_namespace_returns_empty(self):\n        \"\"\"Test that xpath1_filter returns empty on Atom with default namespace (known limitation).\"\"\"\n        result = html_tools.xpath1_filter('//title/text()', atom_feed_with_default_ns, is_xml=True)\n        # xpath1_filter (lxml) doesn't support default namespaces, so this returns empty\n        assert result == ''\n\n    def test_xpath1_filter_local_name_workaround(self):\n        \"\"\"Test that xpath1_filter works with local-name() workaround on Atom feeds.\"\"\"\n        result = html_tools.xpath1_filter('//*[local-name()=\"title\"]/text()', atom_feed_with_default_ns, is_xml=True)\n        assert 'Release notes from PowerToys' in result\n        assert 'Release 0.95.1' in result\n\n\nif __name__ == '__main__':\n    pytest.main([__file__, '-v'])\n"
  },
  {
    "path": "changedetectionio/tests/test_xpath_selector.py",
    "content": "# -*- coding: utf-8 -*-\n\n\nfrom flask import url_for\nfrom .util import  wait_for_all_checks, delete_all_watches\nfrom ..processors.magic import RSS_XML_CONTENT_TYPES\nimport os\n\n\ndef set_rss_atom_feed_response(datastore_path, header='', ):\n    test_return_data = f\"\"\"{header}\n\n\n    \n    RSS Feed\n    \n        \n    \n    \n        \n    \n    en-us\n    water News RSS\n    \n         🍁 Lets go discount\n        
    ok heres the description
    \n        \n        \n        \n        Wed, 08 Oct 2025 15:28:55 +0000\n        https://store.waterpowered.com/news/app/1643320/view/511845698831908921\n        \n        \n    \n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n\n    return None\n\n\n\ndef set_original_response(datastore_path):\n    test_return_data = \"\"\"\n       \n     Some initial text
    \n     
    Which is across multiple lines
    \n     
    \n     So let's see what happens.  
    \n     
    Some text thats the same
    \n     
    Some text that will change
    \n     \n     \n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n    return None\n\n\ndef set_modified_response(datastore_path):\n    test_return_data = \"\"\"\n       \n     Some initial text
    \n     
    Which is across multiple lines
    \n     
    \n     So let's see what happens.  THIS CHANGES AND SHOULDNT TRIGGER A CHANGE
    \n     
    Some text thats the same
    \n     
    Some new text
    \n     \n     \n    \"\"\"\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(test_return_data)\n\n    return None\n\n\n# Handle utf-8 charset replies https://github.com/dgtlmoon/changedetection.io/pull/613\ndef test_check_xpath_filter_utf8(client, live_server, measure_memory_usage, datastore_path):\n    filter = '//item/*[self::description]'\n\n    d = '''\n\n\t\n\t\trpilocator.com\n\t\thttps://rpilocator.com\n\t\tFind Raspberry Pi Computers in Stock\n\t\tThu, 19 May 2022 23:27:30 GMT\n\t\t\n\t\t\thttps://rpilocator.com/favicon.png\n\t\t\trpilocator.com\n\t\t\thttps://rpilocator.com/\n\t\t\t32\n\t\t\t32\n\t\t\n\t\t\n\t\t\tStock Alert (UK): RPi CM4 - 1GB RAM, No MMC, No Wifi is In Stock at Pimoroni\n\t\t\tStock Alert (UK): RPi CM4 - 1GB RAM, No MMC, No Wifi is In Stock at Pimoroni\n\t\t\thttps://rpilocator.com?vendor=pimoroni&utm_source=feed&utm_medium=rss\n\t\t\tpimoroni\n\t\t\tUK\n\t\t\tCM4\n\t\t\tF9FAB0D9-DF6F-40C8-8DEE5FC0646BB722\n\t\t\tThu, 19 May 2022 14:32:32 GMT\n\t\t\n\t\n'''\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(d)\n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True, content_type=\"application/rss+xml;charset=UTF-8\")\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={\"include_filters\": filter, \"url\": test_url, \"tags\": \"\", \"headers\": \"\", 'fetch_backend': \"html_requests\", \"time_between_check_use_default\": \"y\"},\n        follow_redirects=True\n    )\n    assert b\"Updated watch.\" in res.data\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'Unicode strings with encoding declaration are not supported.' not in res.data\n    delete_all_watches(client)\n\n\n# Handle utf-8 charset replies https://github.com/dgtlmoon/changedetection.io/pull/613\ndef test_check_xpath_text_function_utf8(client, live_server, measure_memory_usage, datastore_path):\n    filter = '//item/title/text()'\n\n    d = '''\n\n\t\n\t\trpilocator.com\n\t\thttps://rpilocator.com\n\t\tFind Raspberry Pi Computers in Stock\n\t\tThu, 19 May 2022 23:27:30 GMT\n\t\t\n\t\t\thttps://rpilocator.com/favicon.png\n\t\t\trpilocator.com\n\t\t\thttps://rpilocator.com/\n\t\t\t32\n\t\t\t32\n\t\t\n\t\t\n\t\t\tStock Alert (UK): RPi CM4\n\t\t\tsomething else unrelated\n\t\t\n\t\t\n\t\t\tStock Alert (UK): Big monitor\n\t\t\tsomething else unrelated\n\t\t\t\t\n\t\n'''\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(d)\n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True, content_type=\"application/rss+xml;charset=UTF-8\")\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={\"include_filters\": filter, \"url\": test_url, \"tags\": \"\", \"headers\": \"\", 'fetch_backend': \"html_requests\", \"time_between_check_use_default\": \"y\"},\n        follow_redirects=True\n    )\n    assert b\"Updated watch.\" in res.data\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'Unicode strings with encoding declaration are not supported.' not in res.data\n\n    # The service should echo back the request headers\n    res = client.get(\n        url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"),\n        follow_redirects=True\n    )\n\n    assert b'Stock Alert (UK): RPi CM4' in res.data\n    assert b'Stock Alert (UK): Big monitor' in res.data\n\n    delete_all_watches(client)\n\n\ndef test_check_markup_xpath_filter_restriction(client, live_server, measure_memory_usage, datastore_path):\n    xpath_filter = \"//*[contains(@class, 'sametext')]\"\n\n    set_original_response(datastore_path=datastore_path)\n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    # Give the thread time to pick it up\n    wait_for_all_checks(client)\n\n    # Goto the edit page, add our ignore text\n    # Add our URL to the import page\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={\"include_filters\": xpath_filter, \"url\": test_url, \"tags\": \"\", \"headers\": \"\", 'fetch_backend': \"html_requests\", \"time_between_check_use_default\": \"y\"},\n        follow_redirects=True\n    )\n    assert b\"Updated watch.\" in res.data\n\n    # Give the thread time to pick it up\n    wait_for_all_checks(client)\n\n    # view it/reset state back to viewed\n    client.get(url_for(\"ui.ui_diff.diff_history_page\", uuid=\"first\"), follow_redirects=True)\n\n    #  Make a change\n    set_modified_response(datastore_path=datastore_path)\n\n    # Trigger a check\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    # Give the thread time to pick it up\n    wait_for_all_checks(client)\n\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'has-unread-changes' not in res.data\n    delete_all_watches(client)\n\n\ndef test_xpath_validation(client, live_server, measure_memory_usage, datastore_path):\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={\"include_filters\": \"/something horrible\", \"url\": test_url, \"tags\": \"\", \"headers\": \"\", 'fetch_backend': \"html_requests\", \"time_between_check_use_default\": \"y\"},\n        follow_redirects=True\n    )\n    assert b\"is not a valid XPath expression\" in res.data\n    delete_all_watches(client)\n\n\ndef test_xpath23_prefix_validation(client, live_server, measure_memory_usage, datastore_path):\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={\"include_filters\": \"xpath:/something horrible\", \"url\": test_url, \"tags\": \"\", \"headers\": \"\", 'fetch_backend': \"html_requests\", \"time_between_check_use_default\": \"y\"},\n        follow_redirects=True\n    )\n    assert b\"is not a valid XPath expression\" in res.data\n    delete_all_watches(client)\n\ndef test_xpath1_lxml(client, live_server, measure_memory_usage, datastore_path):\n    \n\n    d = '''\n    \n    \t\n    \t\trpilocator.com\n    \t\thttps://rpilocator.com\n    \t\tFind Raspberry Pi Computers in Stock\n    \t\tThu, 19 May 2022 23:27:30 GMT\n    \t\t\n    \t\t\thttps://rpilocator.com/favicon.png\n    \t\t\trpilocator.com\n    \t\t\thttps://rpilocator.com/\n    \t\t\t32\n    \t\t\t32\n    \t\t\n    \t\t\n    \t\t\tStock Alert (UK): RPi CM4\n    \t\t\tsomething else unrelated\n    \t\t\n    \t\t\n    \t\t\tStock Alert (UK): Big monitorěěěě\n    \t\t\tsomething else unrelated\n    \t\t\t\t\n    \t\n    '''.encode('utf-8')\n\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"wb\") as f:\n        f.write(d)\n\n\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={\"include_filters\": \"xpath1://title/text()\", \"url\": test_url, \"tags\": \"\", \"headers\": \"\",\n              'fetch_backend': \"html_requests\", \"time_between_check_use_default\": \"y\"},\n        follow_redirects=True\n    )\n\n    ##### #2312\n    wait_for_all_checks(client)\n    res = client.get(url_for(\"watchlist.index\"))\n    assert b'_ElementStringResult' not in res.data # tested with 5.1.1 when it was removed and 5.1.0\n    assert b'Exception' not in res.data\n    res = client.get(\n        url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"),\n        follow_redirects=True\n    )\n\n    assert b\"rpilocator.com\" in res.data  # in selector\n    assert \"Stock Alert (UK): Big monitorěěěě\".encode('utf-8') in res.data  # not in selector\n\n    #####\n\n\ndef test_xpath1_validation(client, live_server, measure_memory_usage, datastore_path):\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={\"include_filters\": \"xpath1:/something horrible\", \"url\": test_url, \"tags\": \"\", \"headers\": \"\", 'fetch_backend': \"html_requests\", \"time_between_check_use_default\": \"y\"},\n        follow_redirects=True\n    )\n    assert b\"is not a valid XPath expression\" in res.data\n    delete_all_watches(client)\n\n\n# actually only really used by the distll.io importer, but could be handy too\ndef test_check_with_prefix_include_filters(client, live_server, measure_memory_usage, datastore_path):\n    delete_all_watches(client)\n\n    set_original_response(datastore_path=datastore_path)\n    wait_for_all_checks(client)\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={\"include_filters\": \"xpath://*[contains(@class, 'sametext')]\", \"url\": test_url, \"tags\": \"\", \"headers\": \"\",\n              'fetch_backend': \"html_requests\", \"time_between_check_use_default\": \"y\"},\n        follow_redirects=True\n    )\n\n    assert b\"Updated watch.\" in res.data\n    wait_for_all_checks(client)\n\n    res = client.get(\n        url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"),\n        follow_redirects=True\n    )\n\n    assert b\"Some text thats the same\" in res.data  # in selector\n    assert b\"Some text that will change\" not in res.data  # not in selector\n\n    delete_all_watches(client)\n\n\ndef test_various_rules(client, live_server, measure_memory_usage, datastore_path):\n    # Just check these don't error\n    ##  live_server_setup(live_server) # Setup on conftest per function\n    with open(os.path.join(datastore_path, \"endpoint-content.txt\"), \"w\") as f:\n        f.write(\"\"\"\n       \n     Some initial text
    \n     
    Which is across multiple lines
    \n     
    \n     So let's see what happens.  
    \n     
    Some text thats the same
    \n     
    Some text that will change
    \n     some linky \n     another some linky \n     \n          \n     \n     \n    \"\"\")\n\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    for r in ['//div', '//a', 'xpath://div', 'xpath://a']:\n        res = client.post(\n            url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n            data={\"include_filters\": r,\n                  \"url\": test_url,\n                  \"tags\": \"\",\n                  \"headers\": \"\",\n                  'fetch_backend': \"html_requests\",\n                  \"time_between_check_use_default\": \"y\"},\n            follow_redirects=True\n        )\n        wait_for_all_checks(client)\n        assert b\"Updated watch.\" in res.data\n        res = client.get(url_for(\"watchlist.index\"))\n        assert b'fetch-error' not in res.data, f\"Should not see errors after '{r} filter\"\n\n    delete_all_watches(client)\n\n\ndef test_xpath_20(client, live_server, measure_memory_usage, datastore_path):\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    set_original_response(datastore_path=datastore_path)\n\n    test_url = url_for('test_endpoint', _external=True)\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=uuid),\n        data={\"include_filters\": \"//*[contains(@class, 'sametext')]|//*[contains(@class, 'changetext')]\",\n              \"url\": test_url,\n              \"tags\": \"\",\n              \"headers\": \"\",\n              'fetch_backend': \"html_requests\",\n              \"time_between_check_use_default\": \"y\"},\n        follow_redirects=True\n    )\n\n    assert b\"Updated watch.\" in res.data\n    wait_for_all_checks(client)\n\n    res = client.get(\n        url_for(\"ui.ui_preview.preview_page\", uuid=uuid),\n        follow_redirects=True\n    )\n\n    assert b\"Some text thats the same\" in res.data  # in selector\n    assert b\"Some text that will change\" in res.data  # in selector\n\n    delete_all_watches(client)\n\n\ndef test_xpath_20_function_count(client, live_server, measure_memory_usage, datastore_path):\n    set_original_response(datastore_path=datastore_path)\n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={\"include_filters\": \"xpath:count(//div) * 123456789987654321\",\n              \"url\": test_url,\n              \"tags\": \"\",\n              \"headers\": \"\",\n              'fetch_backend': \"html_requests\",\n              \"time_between_check_use_default\": \"y\"},\n        follow_redirects=True\n    )\n\n    assert b\"Updated watch.\" in res.data\n    wait_for_all_checks(client)\n\n    res = client.get(\n        url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"),\n        follow_redirects=True\n    )\n\n    assert b\"246913579975308642\" in res.data  # in selector\n\n    delete_all_watches(client)\n\n\ndef test_xpath_20_function_count2(client, live_server, measure_memory_usage, datastore_path):\n    set_original_response(datastore_path=datastore_path)\n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n        data={\"include_filters\": \"/html/body/count(div) * 123456789987654321\",\n              \"url\": test_url,\n              \"tags\": \"\",\n              \"headers\": \"\",\n              'fetch_backend': \"html_requests\",\n              \"time_between_check_use_default\": \"y\"},\n        follow_redirects=True\n    )\n\n    assert b\"Updated watch.\" in res.data\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n\n    wait_for_all_checks(client)\n\n    res = client.get(\n        url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"),\n        follow_redirects=True\n    )\n\n    assert b\"246913579975308642\" in res.data  # in selector\n\n    delete_all_watches(client)\n\n\ndef test_xpath_20_function_string_join_matches(client, live_server, measure_memory_usage, datastore_path):\n    set_original_response(datastore_path=datastore_path)\n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', _external=True)\n    uuid = client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=uuid),\n        data={\n            \"include_filters\": \"xpath:string-join(//*[contains(@class, 'sametext')]|//*[matches(@class, 'changetext')], 'specialconjunction')\",\n            \"url\": test_url,\n            \"tags\": \"\",\n            \"headers\": \"\",\n            'fetch_backend': \"html_requests\",\n            \"time_between_check_use_default\": \"y\"},\n        follow_redirects=True\n    )\n\n    assert b\"Updated watch.\" in res.data\n    wait_for_all_checks(client)\n\n    res = client.get(\n        url_for(\"ui.ui_preview.preview_page\", uuid=uuid),\n        follow_redirects=True\n    )\n\n    assert b\"Some text thats the samespecialconjunctionSome text that will change\" in res.data  # in selector\n\n    delete_all_watches(client)\n\n\ndef _subtest_xpath_rss(client, datastore_path, content_type='text/html'):\n\n    # Add our URL to the import page\n    test_url = url_for('test_endpoint', content_type=content_type, _external=True)\n    res = client.post(\n        url_for(\"ui.ui_views.form_quick_watch_add\"),\n        data={\"url\": test_url, \"tags\": '', 'edit_and_watch_submit_button': 'Edit > Watch'},\n        follow_redirects=True\n    )\n\n    assert b\"Watch added in Paused state, saving will unpause\" in res.data\n\n    res = client.post(\n        url_for(\"ui.ui_edit.edit_page\", uuid=\"first\", unpause_on_save=1),\n        data={\n            \"url\": test_url,\n            \"include_filters\": \"xpath://item\",\n            \"tags\": '',\n            \"fetch_backend\": \"html_requests\",\n            \"time_between_check_use_default\": \"y\",\n        },\n        follow_redirects=True\n    )\n\n    assert b\"unpaused\" in res.data\n    wait_for_all_checks(client)\n\n    res = client.get(\n        url_for(\"ui.ui_preview.preview_page\", uuid=\"first\"),\n        follow_redirects=True\n    )\n\n    assert b\"Lets go discount\" in res.data, f\"When testing for Lets go discount called with content type '{content_type}'\"\n    assert b\"Events and Announcements\" not in res.data, f\"When testing for Lets go discount called with content type '{content_type}'\" # It should not be here because thats not our selector target\n\n    delete_all_watches(client)\n\n# Be sure all-in-the-wild types of RSS feeds work with xpath\ndef test_rss_xpath(client, live_server, measure_memory_usage, datastore_path):\n    for feed_header in ['', '']:\n        set_rss_atom_feed_response(header=feed_header, datastore_path=datastore_path)\n        for content_type in RSS_XML_CONTENT_TYPES:\n            _subtest_xpath_rss(client, content_type=content_type, datastore_path=datastore_path)\n\n\n# GHSA-6fmw-82m7-jq6p — XPath arbitrary file read via unparsed-text() and friends\n# Unit-level: verify xpath_filter() and SafeXPath3Parser block all dangerous functions.\ndef test_xpath_blocked_functions_unit():\n    \"\"\"Dangerous XPath 3.0 functions must be rejected at the parser level (no live server needed).\"\"\"\n    import elementpath\n    from changedetectionio.html_tools import xpath_filter, SafeXPath3Parser\n    from lxml import html\n\n    html_content = '
    safe content
    '\n\n    dangerous_expressions = [\n        \"unparsed-text('file:///etc/passwd')\",\n        \"unparsed-text-lines('file:///etc/passwd')\",\n        \"unparsed-text-available('file:///etc/passwd')\",\n        \"doc('file:///etc/passwd')\",\n        \"doc-available('file:///etc/passwd')\",\n        \"environment-variable('PATH')\",\n        \"available-environment-variables()\",\n    ]\n\n    for expr in dangerous_expressions:\n        # xpath_filter() must raise, not silently return file contents\n        try:\n            result = xpath_filter(expr, html_content)\n            assert False, f\"xpath_filter should have raised for: {expr!r}, got: {result!r}\"\n        except elementpath.ElementPathError:\n            pass  # expected\n\n        # SafeXPath3Parser must reject the expression at parse time\n        tree = html.fromstring(html_content)\n        try:\n            elementpath.select(tree, expr, parser=SafeXPath3Parser)\n            assert False, f\"SafeXPath3Parser should have raised for: {expr!r}\"\n        except elementpath.ElementPathError:\n            pass  # expected\n\n    # Sanity check: normal XPath still works\n    result = xpath_filter('//p/text()', html_content)\n    assert result == 'safe content'\n\n\n# GHSA-6fmw-82m7-jq6p — form validation must also reject dangerous XPath expressions.\ndef test_xpath_blocked_functions_form_validation(client, live_server, measure_memory_usage, datastore_path):\n    \"\"\"Edit-form validation must reject dangerous XPath 3.0 functions before they are stored.\"\"\"\n    from flask import url_for\n\n    set_original_response(datastore_path=datastore_path)\n    test_url = url_for('test_endpoint', _external=True)\n    client.application.config.get('DATASTORE').add_watch(url=test_url)\n    client.get(url_for(\"ui.form_watch_checknow\"), follow_redirects=True)\n    wait_for_all_checks(client)\n\n    dangerous_expressions = [\n        \"xpath:unparsed-text('file:///etc/passwd')\",\n        \"xpath:environment-variable('PATH')\",\n        \"xpath:doc('file:///etc/passwd')\",\n    ]\n\n    for expr in dangerous_expressions:\n        res = client.post(\n            url_for(\"ui.ui_edit.edit_page\", uuid=\"first\"),\n            data={\"include_filters\": expr, \"url\": test_url, \"tags\": \"\", \"headers\": \"\",\n                  'fetch_backend': \"html_requests\", \"time_between_check_use_default\": \"y\"},\n            follow_redirects=True\n        )\n        assert b\"is not a valid XPath expression\" in res.data, \\\n            f\"Form should reject dangerous expression: {expr!r}\"\n\n    delete_all_watches(client)\n"
  },
  {
    "path": "changedetectionio/tests/test_xpath_selector_unit.py",
    "content": "import sys\nimport os\nimport pytest\n\nfrom changedetectionio import html_tools\n\nsys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))\n\n# test generation guide.\n# 1. Do not include encoding in the xml declaration if the test object is a str type.\n# 2. Always paraphrase test.\n\nhotels = \"\"\"\n\n  \n    \n      Christopher\n      Anderson\n      25\n    \n    \n      Christopher\n      Carter\n      30\n    \n  \n  \n    \n      Lisa\n      Walker\n      60\n    \n    \n      Jessica\n      Walker\n      32\n    \n    \n      Jennifer\n      Roberts\n      50\n    \n  \n\"\"\"\n\n@pytest.mark.parametrize(\"html_content\", [hotels])\n@pytest.mark.parametrize(\"xpath, answer\", [('(//staff/given_name, //staff/age)', '25'),\n                          (\"xs:date('2023-10-10')\", '2023-10-10'),\n                          (\"if (/hotel/branch[@location = 'California']/staff[1]/age = 25) then 'is 25' else 'is not 25'\", 'is 25'),\n                          (\"if (//hotel/branch[@location = 'California']/staff[1]/age = 25) then 'is 25' else 'is not 25'\", 'is 25'),\n                          (\"if (count(/hotel/branch/staff) = 5) then true() else false()\", 'true'),\n                          (\"if (count(//hotel/branch/staff) = 5) then true() else false()\", 'true'),\n                          (\"for $i in /hotel/branch/staff return if ($i/age >= 40) then upper-case($i/surname) else lower-case($i/surname)\", 'anderson'),\n                          (\"given_name  =  'Christopher' and age  =  40\", 'false'),\n                          (\"//given_name  =  'Christopher' and //age  =  40\", 'false'),\n                          #(\"(staff/given_name, staff/age)\", 'Lisa'),\n                          (\"(//staff/given_name, //staff/age)\", 'Lisa'),\n                          #(\"hotel/branch[@location = 'California']/staff/age union hotel/branch[@location = 'Las Vegas']/staff/age\", ''),\n                          (\"(//hotel/branch[@location = 'California']/staff/age union //hotel/branch[@location = 'Las Vegas']/staff/age)\", '60'),\n                          (\"(200 to 210)\", \"205\"),\n                          (\"(//hotel/branch[@location = 'California']/staff/age union //hotel/branch[@location = 'Las Vegas']/staff/age)\", \"50\"),\n                          (\"(1, 9, 9, 5)\", \"5\"),\n                          (\"(3, (), (14, 15), 92, 653)\", \"653\"),\n                          (\"for $i in /hotel/branch/staff return $i/given_name\", \"Christopher\"),\n                          (\"for $i in //hotel/branch/staff return $i/given_name\", \"Christopher\"),\n                          (\"distinct-values(for $i in /hotel/branch/staff return $i/given_name)\", \"Jessica\"),\n                          (\"distinct-values(for $i in //hotel/branch/staff return $i/given_name)\", \"Jessica\"),\n                          (\"for $i in (7 to  15) return $i*10\", \"130\"),\n                          (\"some $i in /hotel/branch/staff satisfies $i/age < 20\", \"false\"),\n                          (\"some $i in //hotel/branch/staff satisfies $i/age < 20\", \"false\"),\n                          (\"every $i in /hotel/branch/staff satisfies $i/age > 20\", \"true\"),\n                          (\"every $i in //hotel/branch/staff satisfies $i/age > 20 \", \"true\"),\n                          (\"let $x := branch[@location = 'California'], $y := branch[@location = 'Las Vegas'] return (avg($x/staff/age), avg($y/staff/age))\", \"27.5\"),\n                          (\"let $x := //branch[@location = 'California'], $y := //branch[@location = 'Las Vegas'] return (avg($x/staff/age), avg($y/staff/age))\", \"27.5\"),\n                          (\"let $nu := 1, $de := 1000 return  'probability = ' || $nu div $de * 100 || '%'\", \"0.1%\"),\n                          (\"let $nu := 2, $probability := function ($argument) { 'probability = ' ||  $nu div $argument  * 100 || '%'}, $de := 5 return $probability($de)\", \"40%\"),\n                          (\"'XPATH2.0-3.1 dissemination' instance of xs:string \", \"true\"),\n                          (\"'new stackoverflow question incoming' instance of xs:integer \", \"false\"),\n                          (\"'50000' cast as xs:integer\", \"50000\"),\n                          (\"//branch[@location = 'California']/staff[1]/surname eq 'Anderson'\", \"true\"),\n                          (\"fn:false()\", \"false\")])\ndef test_hotels(html_content, xpath, answer):\n    html_content = html_tools.xpath_filter(xpath, html_content, append_pretty_line_formatting=True)\n    assert type(html_content) == str\n    assert answer in html_content\n\n\n\nbranches_to_visit = \"\"\"\n  \n     \n         Area 51\n         A place with no name\n         Stalsk12\n     \n      \n         Stalsk12\n         Barcelona\n         Paris\n     \n \"\"\"\n@pytest.mark.parametrize(\"html_content\", [branches_to_visit])\n@pytest.mark.parametrize(\"xpath, answer\", [\n    (\"manager[@name = 'Godot']/branch union manager[@name = 'Freya']/branch\", \"Area 51\"),\n    (\"//manager[@name = 'Godot']/branch union //manager[@name = 'Freya']/branch\", \"Stalsk12\"),\n    (\"manager[@name = 'Godot']/branch | manager[@name = 'Freya']/branch\", \"Stalsk12\"),\n    (\"//manager[@name = 'Godot']/branch | //manager[@name = 'Freya']/branch\", \"Stalsk12\"),\n    (\"manager/branch intersect manager[@name = 'Godot']/branch\", \"A place with no name\"),\n    (\"//manager/branch intersect //manager[@name = 'Godot']/branch\", \"A place with no name\"),\n    (\"manager[@name = 'Godot']/branch intersect manager[@name = 'Freya']/branch\", \"\"),\n    (\"manager/branch except manager[@name = 'Godot']/branch\", \"Barcelona\"),\n    (\"manager[@name = 'Godot']/branch[1]  eq 'Area 51'\", \"true\"),\n    (\"//manager[@name = 'Godot']/branch[1]  eq 'Area 51'\", \"true\"),\n    (\"manager[@name = 'Godot']/branch[1]  eq 'Seoul'\", \"false\"),\n    (\"//manager[@name = 'Godot']/branch[1]  eq 'Seoul'\", \"false\"),\n    (\"manager[@name = 'Godot']/branch[2] eq manager[@name = 'Freya']/branch[2]\", \"false\"),\n    (\"//manager[@name = 'Godot']/branch[2] eq //manager[@name = 'Freya']/branch[2]\", \"false\"),\n    (\"manager[1]/@room_no lt manager[2]/@room_no\", \"false\"),\n    (\"//manager[1]/@room_no lt //manager[2]/@room_no\", \"false\"),\n    (\"manager[1]/@room_no gt manager[2]/@room_no\", \"true\"),\n    (\"//manager[1]/@room_no gt //manager[2]/@room_no\", \"true\"),\n    (\"manager[@name = 'Godot']/branch[1]  = 'Area 51'\", \"true\"),\n    (\"//manager[@name = 'Godot']/branch[1]  = 'Area 51'\", \"true\"),\n    (\"manager[@name = 'Godot']/branch[1]  = 'Seoul'\", \"false\"),\n    (\"//manager[@name = 'Godot']/branch[1]  = 'Seoul'\", \"false\"),\n    (\"manager[@name = 'Godot']/branch  = 'Area 51'\", \"true\"),\n    (\"//manager[@name = 'Godot']/branch  = 'Area 51'\", \"true\"),\n    (\"manager[@name = 'Godot']/branch  = 'Barcelona'\", \"false\"),\n    (\"//manager[@name = 'Godot']/branch  = 'Barcelona'\", \"false\"),\n    (\"manager[1]/@room_no > manager[2]/@room_no\", \"true\"),\n    (\"//manager[1]/@room_no > //manager[2]/@room_no\", \"true\"),\n    (\"manager[@name = 'Godot']/branch[ . = 'Stalsk12'] is manager[1]/branch[1]\", \"false\"),\n    (\"//manager[@name = 'Godot']/branch[ . = 'Stalsk12'] is //manager[1]/branch[1]\", \"false\"),\n    (\"manager[@name = 'Godot']/branch[ . = 'Stalsk12'] is manager[1]/branch[3]\", \"true\"),\n    (\"//manager[@name = 'Godot']/branch[ . = 'Stalsk12'] is //manager[1]/branch[3]\", \"true\"),\n    (\"manager[@name = 'Godot']/branch[ . = 'Stalsk12'] <<  manager[1]/branch[1]\", \"false\"),\n    (\"//manager[@name = 'Godot']/branch[ . = 'Stalsk12'] <<  //manager[1]/branch[1]\", \"false\"),\n    (\"manager[@name = 'Godot']/branch[ . = 'Stalsk12']  >>  manager[1]/branch[1]\", \"true\"),\n    (\"//manager[@name = 'Godot']/branch[ . = 'Stalsk12'] >>  //manager[1]/branch[1]\", \"true\"),\n    (\"manager[@name = 'Godot']/branch[ . = 'Stalsk12'] is manager[@name = 'Freya']/branch[ . = 'Stalsk12']\", \"false\"),\n    (\"//manager[@name = 'Godot']/branch[ . = 'Stalsk12'] is //manager[@name = 'Freya']/branch[ . = 'Stalsk12']\", \"false\"),\n    (\"manager[1]/@name || manager[2]/@name\", \"GodotFreya\"),\n    (\"//manager[1]/@name || //manager[2]/@name\", \"GodotFreya\"),\n                          ])\ndef test_branches_to_visit(html_content, xpath, answer):\n    html_content = html_tools.xpath_filter(xpath, html_content, append_pretty_line_formatting=True)\n    assert type(html_content) == str\n    assert answer in html_content\n\ntrips = \"\"\"\n\n   \n       2023-10-06\n       2023-10-10\n       \n           4\n           2000.00\n       \n   \n   \n       2023-10-06\n       2023-10-12\n       \n           6\n           3500.34\n       \n   \n\"\"\"\n@pytest.mark.parametrize(\"html_content\", [trips])\n@pytest.mark.parametrize(\"xpath, answer\", [\n    (\"1 + 9 * 9 + 5 div 5\", \"83\"),\n    (\"(1 + 9 * 9 + 5) div 6\", \"14.5\"),\n    (\"23 idiv 3\", \"7\"),\n    (\"23 div 3\", \"7.66666666\"),\n    (\"for $i in ./trip return $i/traveler/duration * $i/traveler/price\", \"21002.04\"),\n    (\"for $i in ./trip return $i/traveler/duration \", \"4\"),\n    (\"for $i in .//trip return $i/traveler/duration * $i/traveler/price\", \"21002.04\"),\n    (\"sum(for $i in ./trip return $i/traveler/duration * $i/traveler/price)\", \"29002.04\"),\n    (\"sum(for $i in .//trip return $i/traveler/duration * $i/traveler/price)\", \"29002.04\"),\n    #(\"trip[1]/depart - trip[1]/arrive\", \"fail_to_get_answer\"),\n    #(\"//trip[1]/depart - //trip[1]/arrive\", \"fail_to_get_answer\"),\n    #(\"trip[1]/depart + trip[1]/arrive\", \"fail_to_get_answer\"),\n    #(\"xs:date(trip[1]/depart) + xs:date(trip[1]/arrive)\", \"fail_to_get_answer\"),\n    (\"(//trip[1]/arrive cast as xs:date) - (//trip[1]/depart cast as xs:date)\", \"P4D\"),\n    (\"(//trip[1]/depart cast as xs:date) - (//trip[1]/arrive cast as xs:date)\", \"-P4D\"),\n    (\"(//trip[1]/depart cast as xs:date) + xs:dayTimeDuration('P3D')\", \"2023-10-09\"),\n    (\"(//trip[1]/depart cast as xs:date) - xs:dayTimeDuration('P3D')\", \"2023-10-03\"),\n    (\"(456, 623) instance of xs:integer\", \"false\"),\n    (\"(456, 623) instance of xs:integer*\", \"true\"),\n    (\"/trips/trip instance of element()\", \"false\"),\n    (\"/trips/trip instance of element()*\", \"true\"),\n    (\"/trips/trip[1]/arrive instance of xs:date\", \"false\"),\n    (\"date(/trips/trip[1]/arrive) instance of xs:date\", \"true\"),\n    (\"'8' cast as xs:integer\", \"8\"),\n    (\"'11.1E3' cast as xs:double\", \"11100\"),\n    (\"6.5 cast as xs:integer\", \"6\"),\n    #(\"/trips/trip[1]/arrive cast as xs:dateTime\", \"fail_to_get_answer\"),\n    (\"/trips/trip[1]/arrive cast as xs:date\", \"2023-10-10\"),\n    (\"('2023-10-12') cast as xs:date\", \"2023-10-12\"),\n    (\"for $i in //trip return concat($i/depart, '  ', $i/arrive)\", \"2023-10-06  2023-10-10\"),\n                          ])\ndef test_trips(html_content, xpath, answer):\n    html_content = html_tools.xpath_filter(xpath, html_content, append_pretty_line_formatting=True)\n    assert type(html_content) == str\n    assert answer in html_content\n\n\n# Test for UTF-8 encoding bug fix (issue #3658)\n# Polish and other UTF-8 characters should be preserved correctly\npolish_html = \"\"\"\n\n\n\n
    \n    \n        Naukowcy potwierdzają: oglądanie krótkich filmików prowadzi do \"zgnilizny mózgu\"\n    \n
    \n\n
    \n    
    Cyrillic: Привет мир
    \n    
    Greek: Γειά σου κόσμε
    \n    
    Arabic: مرحبا بالعالم
    \n    
    Chinese: 你好世界
    \n    
    Japanese: こんにちは世界
    \n    
    Emoji: 🌍🎉✨
    \n
    \n\n\n\"\"\"\n\n\n@pytest.mark.parametrize(\"html_content\", [polish_html])\n@pytest.mark.parametrize(\"xpath, expected_text\", [\n    # Test Polish characters in xpath_filter\n    ('//a[(contains(@class,\"index--s-headline-link\"))]', 'Naukowcy potwierdzają'),\n    ('//a[(contains(@class,\"index--s-headline-link\"))]', 'oglądanie krótkich filmików'),\n    ('//a[(contains(@class,\"index--s-headline-link\"))]', 'zgnilizny mózgu'),\n    ('//a[@class=\"other-class\"]', 'żółć ąę śń'),\n\n    # Test various Unicode scripts\n    ('//p[@class=\"unicode-test\"]', 'Привет мир'),\n    ('//p[@class=\"unicode-test\"]', 'Γειά σου κόσμε'),\n    ('//p[@class=\"unicode-test\"]', 'مرحبا بالعالم'),\n    ('//p[@class=\"unicode-test\"]', '你好世界'),\n    ('//p[@class=\"unicode-test\"]', 'こんにちは世界'),\n    ('//p[@class=\"unicode-test\"]', '🌍🎉✨'),\n\n    # Test with text() extraction\n    ('//a[@class=\"other-class\"]/text()', 'żółć'),\n])\ndef test_xpath_utf8_encoding(html_content, xpath, expected_text):\n    \"\"\"Test that XPath filters preserve UTF-8 characters correctly (issue #3658)\"\"\"\n    result = html_tools.xpath_filter(xpath, html_content, append_pretty_line_formatting=False)\n    assert type(result) == str\n    assert expected_text in result\n    # Ensure characters are NOT HTML-entity encoded\n    # For example, 'ą' should NOT become 'ą'\n    assert '&#' not in result or expected_text in result\n\n\n@pytest.mark.parametrize(\"html_content\", [polish_html])\n@pytest.mark.parametrize(\"xpath, expected_text\", [\n    # Test Polish characters in xpath1_filter\n    ('//a[(contains(@class,\"index--s-headline-link\"))]', 'Naukowcy potwierdzają'),\n    ('//a[(contains(@class,\"index--s-headline-link\"))]', 'mózgu'),\n    ('//a[@class=\"other-class\"]', 'żółć ąę śń'),\n\n    # Test various Unicode scripts with xpath1\n    ('//p[@class=\"unicode-test\" and contains(text(), \"Cyrillic\")]', 'Привет мир'),\n    ('//p[@class=\"unicode-test\" and contains(text(), \"Greek\")]', 'Γειά σου'),\n    ('//p[@class=\"unicode-test\" and contains(text(), \"Chinese\")]', '你好世界'),\n])\ndef test_xpath1_utf8_encoding(html_content, xpath, expected_text):\n    \"\"\"Test that XPath1 filters preserve UTF-8 characters correctly\"\"\"\n    result = html_tools.xpath1_filter(xpath, html_content, append_pretty_line_formatting=False)\n    assert type(result) == str\n    assert expected_text in result\n    # Ensure characters are NOT HTML-entity encoded\n    assert '&#' not in result or expected_text in result\n\n\n# Test with real-world example from wyborcza.pl (issue #3658)\nwyborcza_style_html = \"\"\"\n\n\n\n
    \n    \n        Naukowcy potwierdzają: oglądanie krótkich filmików prowadzi do \"zgnilizny mózgu\"\n    \n    \n        Zmiany klimatyczne wpływają na życie w miastach\n    \n    \n        Łódź: Nowe inwestycje w infrastrukturę miejską\n    \n
    \n\n\n\"\"\"\n\n\ndef test_wyborcza_real_world_example():\n    \"\"\"Test real-world case from wyborcza.pl that was failing (issue #3658)\"\"\"\n    xpath = '//a[(contains(@class,\"index--s-headline-link\"))]'\n    result = html_tools.xpath_filter(xpath, wyborcza_style_html, append_pretty_line_formatting=False)\n\n    # These exact strings should appear in the result\n    assert 'Naukowcy potwierdzają' in result\n    assert 'oglądanie krótkich filmików' in result\n    assert 'zgnilizny mózgu' in result\n    assert 'Łódź' in result\n\n    # Make sure they're NOT corrupted to mojibake like \"potwierdzajÄ\"\n    assert 'potwierdzajÄ' not in result\n    assert 'oglądanie' not in result\n    assert 'mózgu' not in result\n"
  },
  {
    "path": "changedetectionio/tests/unit/__init__.py",
    "content": "\"\"\"Unit tests for the app.\"\"\"\n"
  },
  {
    "path": "changedetectionio/tests/unit/test-content/README.md",
    "content": "# What is this?\nThis is test content for the python diff engine, we use the JS interface for the front end, because you can explore \ndifferences in words etc, but we use (at the moment) the python difflib engine.\n\nThis content `before.txt` and `after.txt` is for unit testing\n"
  },
  {
    "path": "changedetectionio/tests/unit/test-content/after-2.txt",
    "content": "After twenty years, as cursed as I may be\nok\nand insure that I'm one of those computer nerds.\n"
  },
  {
    "path": "changedetectionio/tests/unit/test-content/after.txt",
    "content": "After twenty years, as cursed as I may be\nfor having learned computerese,\nI continue to examine bits, bytes and words\nxok\nnext-x-ok\nand insure that I'm one of those computer nerds.\nand something new"
  },
  {
    "path": "changedetectionio/tests/unit/test-content/before.txt",
    "content": "After twenty years, as cursed as I may be\nfor having learned computerese,\nI continue to examine bits, bytes and words\nok\nand insure that I'm one of those computer nerds.\n"
  },
  {
    "path": "changedetectionio/tests/unit/test_conditions.py",
    "content": "from changedetectionio.conditions import execute_ruleset_against_all_plugins\nfrom changedetectionio.model import CONDITIONS_MATCH_LOGIC_DEFAULT\nfrom changedetectionio.store import ChangeDetectionStore\nimport shutil\nimport tempfile\nimport time\nimport unittest\nimport uuid\n\n\nclass TestTriggerConditions(unittest.TestCase):\n    def setUp(self):\n\n        # Create a temporary directory for the test datastore\n        self.test_datastore_path = tempfile.mkdtemp()\n\n        # Initialize ChangeDetectionStore with our test path and no default watches\n        self.store = ChangeDetectionStore(\n            datastore_path=self.test_datastore_path,\n            include_default_watches=False\n        )\n\n        # Add a test watch\n        watch_url = \"https://example.com\"\n        self.watch_uuid = self.store.add_watch(url=watch_url)\n\n    def tearDown(self):\n      # Clean up the test datastore\n      self.store.stop_thread = True\n      time.sleep(0.5)  # Give thread time to stop\n      shutil.rmtree(self.test_datastore_path)\n\n    def test_conditions_execution_pass(self):\n        # Get the watch object\n        watch = self.store.data['watching'][self.watch_uuid]\n\n        # Create and save a snapshot\n        first_content = \"I saw 100 people at a rock show\"\n        timestamp1 = int(time.time())\n        snapshot_id1 = str(uuid.uuid4())\n        watch.save_history_blob(contents=first_content,\n                                timestamp=timestamp1,\n                                snapshot_id=snapshot_id1)\n\n        # Add another snapshot\n        second_content = \"I saw 200 people at a rock show\"\n        timestamp2 = int(time.time()) + 60\n        snapshot_id2 = str(uuid.uuid4())\n        watch.save_history_blob(contents=second_content,\n                                timestamp=timestamp2,\n                                snapshot_id=snapshot_id2)\n\n        # Verify both snapshots are stored\n        history = watch.history\n        self.assertEqual(len(history), 2)\n\n        # Retrieve and check snapshots\n        #snapshot1 = watch.get_history_snapshot(timestamp=str(timestamp1))\n        #snapshot2 = watch.get_history_snapshot(timestamp=str(timestamp2))\n\n        self.store.data['watching'][self.watch_uuid].update(\n            {\n                \"conditions_match_logic\": CONDITIONS_MATCH_LOGIC_DEFAULT,\n                \"conditions\": [\n                    {\"operator\": \">=\", \"field\": \"extracted_number\", \"value\": \"10\"},\n                    {\"operator\": \"<=\", \"field\": \"extracted_number\", \"value\": \"5000\"},\n                    {\"operator\": \"in\", \"field\": \"page_text\", \"value\": \"rock\"},\n                    #{\"operator\": \"starts_with\", \"field\": \"page_text\", \"value\": \"I saw\"},\n                ]\n            }\n        )\n\n        # ephemeral_data - some data that could exist before the watch saved a new version\n        result = execute_ruleset_against_all_plugins(current_watch_uuid=self.watch_uuid,\n                                                     application_datastruct=self.store.data,\n                                                     ephemeral_data={'text': \"I saw 500 people at a rock show\"})\n\n        # @todo - now we can test that 'Extract number' increased more than X since last time\n        self.assertTrue(result.get('result'))\n\n\nif __name__ == '__main__':\n    unittest.main()\n"
  },
  {
    "path": "changedetectionio/tests/unit/test_html_to_text.py",
    "content": "#!/usr/bin/env python3\n# coding=utf-8\n\n\"\"\"Unit tests for html_tools.html_to_text function.\"\"\"\n\nimport hashlib\nimport threading\nimport unittest\nfrom queue import Queue\n\nfrom changedetectionio.html_tools import html_to_text\n\n\nclass TestHtmlToText(unittest.TestCase):\n    \"\"\"Test html_to_text function for correctness and thread-safety.\"\"\"\n\n    def test_basic_text_extraction(self):\n        \"\"\"Test basic HTML to text conversion.\"\"\"\n        html = '
    Title
    Paragraph text.
    '\n        text = html_to_text(html)\n\n        assert 'Title' in text\n        assert 'Paragraph text.' in text\n        assert '<' not in text  # HTML tags should be stripped\n        assert '>' not in text\n\n    def test_empty_html(self):\n        \"\"\"Test handling of empty HTML.\"\"\"\n        html = ''\n        text = html_to_text(html)\n\n        # Should return empty or whitespace only\n        assert text.strip() == ''\n\n    def test_nested_elements(self):\n        \"\"\"Test extraction from nested HTML elements.\"\"\"\n        html = '''\n        \n            \n                
    \n                    
    Header
    \n                    
    \n                        
    First paragraph
    \n                        
    Second paragraph
    \n                    
    \n                
    \n            \n        \n        '''\n        text = html_to_text(html)\n\n        assert 'Header' in text\n        assert 'First paragraph' in text\n        assert 'Second paragraph' in text\n\n    def test_anchor_tag_rendering(self):\n        \"\"\"Test anchor tag rendering option.\"\"\"\n        html = 'Link text'\n\n        # Without rendering anchors\n        text_without = html_to_text(html, render_anchor_tag_content=False)\n        assert 'Link text' in text_without\n        assert 'https://example.com' not in text_without\n\n        # With rendering anchors\n        text_with = html_to_text(html, render_anchor_tag_content=True)\n        assert 'Link text' in text_with\n        assert 'https://example.com' in text_with or '[Link text]' in text_with\n\n    def test_rss_mode(self):\n        \"\"\"Test RSS mode converts title tags to h1.\"\"\"\n        html = 'RSS TitleContent'\n\n        # is_rss=True should convert  to <h1>\n        text = html_to_text(html, is_rss=True)\n\n        assert 'RSS Title' in text\n        assert 'Content' in text\n\n    def test_special_characters(self):\n        \"\"\"Test handling of special characters and entities.\"\"\"\n        html = '<html><body><p>Test & <special> characters</p></body></html>'\n        text = html_to_text(html)\n\n        # Entities should be decoded\n        assert 'Test &' in text or 'Test &' in text\n        assert 'special' in text\n\n    def test_whitespace_handling(self):\n        \"\"\"Test that whitespace is properly handled.\"\"\"\n        html = '<html><body><p>Line 1</p><p>Line 2</p></body></html>'\n        text = html_to_text(html)\n\n        # Should have some separation between lines\n        assert 'Line 1' in text\n        assert 'Line 2' in text\n        assert text.count('\\n') >= 1  # At least one newline\n\n    def test_deterministic_output(self):\n        \"\"\"Test that the same HTML always produces the same text.\"\"\"\n        html = '<html><body><h1>Test</h1><p>Content here</p></body></html>'\n\n        # Extract text multiple times\n        results = [html_to_text(html) for _ in range(10)]\n\n        # All results should be identical\n        assert len(set(results)) == 1, \"html_to_text should be deterministic\"\n\n    def test_thread_safety_determinism(self):\n        \"\"\"\n        Test that html_to_text produces deterministic output under high concurrency.\n\n        This verifies that lxml's default parser (used by inscriptis.get_text)\n        is thread-safe and produces consistent results when called from multiple\n        threads simultaneously.\n        \"\"\"\n        html = '''\n        <html>\n            <head><title>Test Page\n            \n                
    Main Heading
    \n                
    \n                    
    First paragraph with bold text.
    \n                    
    Second paragraph with italic text.
    \n                    
      \n                        
    • Item 1
      • \n                        
    • Item 2
      • \n                        
    • Item 3
      • \n                    
    \n                
    \n            \n        \n        '''\n\n        results_queue = Queue()\n\n        def worker(worker_id, iterations=10):\n            \"\"\"Worker that converts HTML to text multiple times.\"\"\"\n            for i in range(iterations):\n                text = html_to_text(html)\n                md5 = hashlib.md5(text.encode('utf-8')).hexdigest()\n                results_queue.put((worker_id, i, md5))\n\n        # Launch many threads simultaneously\n        num_threads = 50\n        threads = []\n\n        for i in range(num_threads):\n            t = threading.Thread(target=worker, args=(i,))\n            threads.append(t)\n            t.start()\n\n        # Wait for all threads to complete\n        for t in threads:\n            t.join()\n\n        # Collect all MD5 results\n        md5_values = []\n        while not results_queue.empty():\n            _, _, md5 = results_queue.get()\n            md5_values.append(md5)\n\n        # All MD5s should be identical\n        unique_md5s = set(md5_values)\n\n        assert len(unique_md5s) == 1, (\n            f\"Thread-safety issue detected! Found {len(unique_md5s)} different MD5 values: {unique_md5s}. \"\n            \"The thread-local parser fix may not be working correctly.\"\n        )\n\n        print(f\"✓ Thread-safety test passed: {len(md5_values)} conversions, all identical\")\n\n    def test_thread_safety_basic(self):\n        \"\"\"Verify basic thread safety - multiple threads can call html_to_text simultaneously.\"\"\"\n        results = []\n        errors = []\n\n        def worker():\n            \"\"\"Worker that converts HTML to text.\"\"\"\n            try:\n                html = '
    Test
    Content
    '\n                text = html_to_text(html)\n                results.append(text)\n            except Exception as e:\n                errors.append(e)\n\n        # Launch 10 threads simultaneously\n        threads = [threading.Thread(target=worker) for _ in range(10)]\n        for t in threads:\n            t.start()\n        for t in threads:\n            t.join()\n\n        # Should have no errors\n        assert len(errors) == 0, f\"Thread-safety errors occurred: {errors}\"\n\n        # All results should be identical\n        assert len(set(results)) == 1, \"All threads should produce identical output\"\n\n        print(f\"✓ Basic thread-safety test passed: {len(results)} threads, no errors\")\n\n    def test_large_html_with_bloated_head(self):\n        \"\"\"\n        Test that html_to_text can handle large HTML documents with massive  bloat.\n\n        SPAs often dump 10MB+ of styles, scripts, and other bloat into the  section.\n        This can cause inscriptis to silently exit when processing very large documents.\n        The fix strips \\n'\n\n        # Generate massive script block (~5MB)\n        large_script = '\\n'\n\n        # Generate lots of SVG bloat (~3MB)\n        svg_bloat = '\\n' * 50000\n\n        # Generate meta/link tags (~2MB)\n        meta_bloat = '\\n' * 50000\n        link_bloat = '\\n' * 50000\n\n        # Generate HTML comments (~1MB)\n        comment_bloat = '\\n' * 50000\n\n        # Generate noscript bloat\n        noscript_bloat = '\\n' * 10000\n\n        # Build the large HTML document\n        html = f'''\n\n\n    Test Page\n    {large_style}\n    {large_script}\n    {svg_bloat}\n    {meta_bloat}\n    {link_bloat}\n    {comment_bloat}\n    {noscript_bloat}\n\n\n    
    Important Heading
    \n    
    This is the actual content that should be extracted.
    \n    
    \n        
    First paragraph with meaningful text.
    \n        
    Second paragraph with more content.
    \n    
    \n    
    Footer text
    \n\n\n'''\n\n        # Verify the HTML is actually large (should be ~20MB+)\n        html_size_mb = len(html) / (1024 * 1024)\n        assert html_size_mb > 15, f\"HTML should be >15MB, got {html_size_mb:.2f}MB\"\n\n        print(f\"  Testing {html_size_mb:.2f}MB HTML document with bloated head...\")\n\n        # This should not crash or silently exit\n        text = html_to_text(html)\n\n        # Verify we got actual text output (not empty/None)\n        assert text is not None, \"html_to_text returned None\"\n        assert len(text) > 0, \"html_to_text returned empty string\"\n\n        # Verify the actual body content was extracted\n        assert 'Important Heading' in text, \"Failed to extract heading\"\n        assert 'actual content that should be extracted' in text, \"Failed to extract paragraph\"\n        assert 'First paragraph with meaningful text' in text, \"Failed to extract first paragraph\"\n        assert 'Second paragraph with more content' in text, \"Failed to extract second paragraph\"\n        assert 'Footer text' in text, \"Failed to extract footer\"\n\n        # Verify bloat was stripped (output should be tiny compared to input)\n        text_size_kb = len(text) / 1024\n        assert text_size_kb < 1, f\"Output too large ({text_size_kb:.2f}KB), bloat not stripped\"\n\n        # Verify no CSS, script content, or SVG leaked through\n        assert 'color:red' not in text, \"Style content leaked into text output\"\n        assert 'console.log' not in text, \"Script content leaked into text output\"\n        assert ' {text_size_kb:.2f}KB text\")\n\n    def test_body_display_none_spa_pattern(self):\n        \"\"\"\n        Test that html_to_text can extract content from pages with display:none body.\n\n        SPAs (Single Page Applications) often use  to hide content\n        until JavaScript loads and renders the page. inscriptis respects CSS display rules,\n        so without preprocessing, it would skip all content and return only newlines.\n\n        The fix strips display:none and visibility:hidden styles from the body tag before\n        processing, allowing text extraction from client-side rendered applications.\n        \"\"\"\n        # Test case 1: Basic display:none\n        html1 = '''\n\nWhat's New – Fluxguard\n\n    
    Important Heading
    \n    
    This is actual content that should be extracted.
    \n    
    \n        
    First paragraph with meaningful text.
    \n        
    Second paragraph with more content.
    \n    
    \n\n'''\n\n        text1 = html_to_text(html1)\n\n        # Before fix: would return ~33 newlines, len(text) ~= 33\n        # After fix: should extract actual content, len(text) > 100\n        assert len(text1) > 100, f\"Expected substantial text output, got {len(text1)} chars\"\n        assert 'Important Heading' in text1, \"Failed to extract heading from display:none body\"\n        assert 'actual content' in text1, \"Failed to extract paragraph from display:none body\"\n        assert 'First paragraph' in text1, \"Failed to extract nested content\"\n\n        # Should not be mostly newlines\n        newline_ratio = text1.count('\\n') / len(text1)\n        assert newline_ratio < 0.5, f\"Output is mostly newlines ({newline_ratio:.2%}), content not extracted\"\n\n        # Test case 2: visibility:hidden (another hiding pattern)\n        html2 = '
    Hidden Content
    Test paragraph.
    '\n        text2 = html_to_text(html2)\n\n        assert 'Hidden Content' in text2, \"Failed to extract content from visibility:hidden body\"\n        assert 'Test paragraph' in text2, \"Failed to extract paragraph from visibility:hidden body\"\n\n        # Test case 3: Mixed styles (display:none with other CSS)\n        html3 = '
    Mixed style content
    '\n        text3 = html_to_text(html3)\n\n        assert 'Mixed style content' in text3, \"Failed to extract content from body with mixed styles\"\n\n        # Test case 4: Case insensitivity (DISPLAY:NONE uppercase)\n        html4 = '
    Uppercase style
    '\n        text4 = html_to_text(html4)\n\n        assert 'Uppercase style' in text4, \"Failed to handle uppercase DISPLAY:NONE\"\n\n        # Test case 5: Space variations (display: none vs display:none)\n        html5 = '
    With spaces
    '\n        text5 = html_to_text(html5)\n\n        assert 'With spaces' in text5, \"Failed to handle 'display: none' with space\"\n\n        # Test case 6: Body with other attributes (class, id)\n        html6 = '
    With attributes
    '\n        text6 = html_to_text(html6)\n\n        assert 'With attributes' in text6, \"Failed to extract from body with multiple attributes\"\n\n        # Test case 7: Should NOT affect opacity:0 (which doesn't hide from inscriptis)\n        html7 = '
    Transparent content
    '\n        text7 = html_to_text(html7)\n\n        # Opacity doesn't affect inscriptis text extraction, content should be there\n        assert 'Transparent content' in text7, \"Incorrectly stripped opacity:0 style\"\n\n        print(\"  ✓ All display:none body tag tests passed\")\n\n    def test_style_tag_with_svg_data_uri(self):\n        \"\"\"\n        Test that style tags containing SVG data URIs are properly stripped.\n\n        Some WordPress and modern sites embed SVG as data URIs in CSS, which contains\n         and  tags within the style content. The regex must use backreferences\n        to ensure  (not  inside the CSS).\n\n        This was causing errors where the regex would match \n\n\n    
    Test Heading
    \n    
    This is the actual content that should be extracted.
    \n    
    \n        \"Test\n    
    \n\n'''\n\n        # This should not crash and should extract the body content\n        text = html_to_text(html)\n\n        # Verify the actual body content was extracted\n        assert text is not None, \"html_to_text returned None\"\n        assert len(text) > 0, \"html_to_text returned empty string\"\n        assert 'Test Heading' in text, \"Failed to extract heading\"\n        assert 'actual content that should be extracted' in text, \"Failed to extract paragraph\"\n\n        # Verify CSS content was stripped (including the SVG data URI)\n        assert '.wp-block-image' not in text, \"CSS class selector leaked into text\"\n        assert 'mask-image' not in text, \"CSS property leaked into text\"\n        assert 'data:image/svg+xml' not in text, \"SVG data URI leaked into text\"\n        assert 'viewBox' not in text, \"SVG attributes leaked into text\"\n\n        # Verify no broken HTML structure\n        assert '' not in text, \"SVG closing tag leaked into text\"\n\n        print(\"  ✓ Style tag with SVG data URI test passed\")\n\n    def test_style_tag_closes_correctly(self):\n        \"\"\"\n        Test that each tag type (style, script, svg) closes with the correct closing tag.\n\n        Before the fix, the regex used (?:style|script|svg|noscript) for both opening and\n        closing tags, which meant ,  with , etc.\n        \"\"\"\n        # Test nested tags where incorrect matching would break\n        html = '''\n\n\n    \n    \n\n\n    
    Content
    \n    \n    
    After SVG
    \n\n'''\n\n        text = html_to_text(html)\n\n        # Should extract body content\n        assert 'Content' in text, \"Failed to extract heading\"\n        assert 'After SVG' in text, \"Failed to extract content after SVG\"\n\n        # Should strip all style/script/svg content\n        assert 'background:' not in text, \"Style content leaked\"\n        assert 'const svg' not in text, \"Script content leaked\"\n        assert ' inside a JS string must not prematurely end the block.\n\n        This is the classic regex failure mode: the old pattern would find the first \n        inside the JS string literal and stop there, leaving the tail of the script block\n        (plus any following content) exposed as raw text. BS4 parses the HTML correctly.\n        \"\"\"\n        html = '''\n
    Before script
    \n\n
    AFTER SCRIPT
    \n'''\n\n        text = html_to_text(html)\n        assert 'Before script' in text\n        assert 'AFTER SCRIPT' in text\n        # Script internals must not leak\n        assert 'var html' not in text\n        assert 'var also' not in text\n\n    def test_content_sandwiched_between_multiple_body_scripts(self):\n        \"\"\"Content between multiple script/style blocks in the body must all survive.\"\"\"\n        html = '''\n\n
    CONTENT A
    \n\n
    CONTENT B
    \n\n
    CONTENT C
    \n\n
    CONTENT D
    \n'''\n\n        text = html_to_text(html)\n        for label in ['CONTENT A', 'CONTENT B', 'CONTENT C', 'CONTENT D']:\n            assert label in text, f\"'{label}' was eaten by script/style stripping\"\n        assert 'var a' not in text\n        assert 'var b' not in text\n        assert 'color: red' not in text\n        assert 'color: blue' not in text\n\n    def test_unicode_and_international_content_preserved(self):\n        \"\"\"Non-ASCII content (umlauts, CJK, soft hyphens) must survive stripping.\"\"\"\n        html = '''\n\n
    German: Aus\\xadge\\xadbucht! — ANMELDUNG — Fan\\xadday 2026
    \n
    Chinese: \\u6ce8\\u518c
    \n
    Japanese: \\u767b\\u9332
    \n
    Korean: \\ub4f1\\ub85d
    \n
    Emoji: \\U0001f4e2
    \n\n'''\n\n        text = html_to_text(html)\n        assert 'ANMELDUNG' in text\n        assert '\\u6ce8\\u518c' in text   # Chinese\n        assert '\\u767b\\u9332' in text   # Japanese\n        assert '\\ub4f1\\ub85d' in text   # Korean\n\n    def test_style_with_type_attribute_is_stripped(self):\n        \"\"\"\n
    VISIBLE CONTENT
    \n'''\n\n        text = html_to_text(html)\n        assert 'VISIBLE CONTENT' in text\n        assert '.important' not in text\n        assert 'display: none' not in text\n\n    def test_ldjson_script_is_stripped(self):\n        \"\"\"\n
    PRODUCT PAGE
    \n'''\n\n        text = html_to_text(html)\n        assert 'PRODUCT PAGE' in text\n        assert '@type' not in text\n        assert '\"price\"' not in text\n\n    def test_inline_svg_is_stripped_entirely(self):\n        \"\"\"\n        Inline SVG elements in the body are stripped by BS4 before passing to inscriptis.\n        SVGs can be huge (icon libraries, data visualisations) and produce garbage path-data\n        text. The old regex code explicitly stripped ; the BS4 path must do the same.\n        \"\"\"\n        html = '''\n
    Before SVG
    \n\n    \n    \n\n
    After SVG
    \n'''\n\n        text = html_to_text(html)\n        assert 'Before SVG' in text\n        assert 'After SVG' in text\n        assert 'M14 5L7' not in text, \"SVG path data should not appear in text output\"\n        assert 'viewBox' not in text, \"SVG attributes should not appear in text output\"\n\n    def test_tag_inside_json_data_attribute_does_not_eat_content(self):\n        \"\"\"\n        Tags inside JSON data attributes with JS-escaped closing tags must not eat real content.\n\n        Real-world case: Elementor/JetEngine WordPress widgets embed HTML (including SVG icons)\n        inside JSON data attributes like data-slider-atts. The HTML inside is JS-escaped, so\n        closing tags appear as <\\\\/svg> rather than .\n\n        The old regex approach would find  inside the attribute value, then fail to find\n        <\\/svg> as a matching close tag, and scan forward to the next real  in the DOM —\n        eating tens of kilobytes of actual page content in the process.\n        \"\"\"\n        html = '''\n\nTest\n\n
    <\\\\/svg><\\\\/i>"}\">\n
    \n
    \n    
    IMPORTANT CONTENT
    \n    
    This text must not be eaten by the tag-stripping logic.
    \n
    \n\n\n'''\n\n        text = html_to_text(html)\n\n        assert 'IMPORTANT CONTENT' in text, (\n            \"Content after a JS-escaped tag in a data attribute was incorrectly stripped. \"\n            \"The tag-stripping logic is matching  inside attribute values and scanning \"\n            \"forward to the next real closing tag in the DOM.\"\n        )\n        assert 'This text must not be eaten' in text\n\n    def test_script_inside_json_data_attribute_does_not_eat_content(self):\n        \"\"\"Same issue as above but with \n\n'''\n\n        text = html_to_text(html)\n\n        assert 'MUST SURVIVE' in text, (\n            \"Content after a JS-escaped \n  \n\n\n\n  \n      
    API docs by Redocly
    ChangeDetection.io API (0.1.6)
    Download OpenAPI specification:
     URL: https://github.com/dgtlmoon/changedetection.io License: Apache 2.0 
    ChangeDetection.io Web page monitoring and notifications API
    REST API for managing Page watches, Group tags, and Notifications.
    \n
    changedetection.io can be driven by its built in simple API, in the examples below you will also find curl command line and python examples to help you get started faster.
    \n
    Where to find my API key?
    The API key can be easily found under the SETTINGS then API tab of changedetection.io dashboard.
    Simply click the API key to automatically copy it to your clipboard.
    \n
    \"Where
    \n
    Connection URL
    The API can be found at /api/v1/, so for example if you run changedetection.io locally on port 5000, then URL would be http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/history.
    \n
    If you are using the hosted/subscription version of changedetection.io, then the URL is based on your login URL, for example:
    https://<your login url>/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091/history
    \n
    Authentication
    Almost all API requests require some authentication, this is provided as an API Key in the header of the HTTP request.
    \n
    For example: x-api-key: YOUR_API_KEY
    \n
    Watch Management
    Core functionality for managing web page monitors. Create, retrieve, update, and delete individual watches. \nEach watch represents a single URL being monitored for changes, with configurable settings for check intervals, \nnotification preferences, and content filtering options.
    \n
    List all watches 
    Return concise list of available web page change monitors (watches) and basic info
    \n
    Authorizations:
    ApiKeyAuth
    query Parameters
    recheck_all
    string
     Value: "1" 
     
    Set to 1 to force recheck of all watches
    \n
    tag
    string
     
    Tag name to filter results
    \n
    Responses
    Development server
    \n
    http://localhost:5000/api/v1/watch
    Production server
    \n
    https://yourdomain.com/api/v1/watch
    Custom server
    \n
    {protocol}://{host}/api/v1/watch
     Request samples 
    • curl
    • Python
    curl -X GET \"http://localhost:5000/api/v1/watch\" \\\n  -H \"x-api-key: YOUR_API_KEY\"\n
     Response samples 
    • 200
    Content type
    application/json
    {
    • \"095be615-a8ad-4c33-8e9c-c7612fbf6c9f\": {
      • \"uuid\": "095be615-a8ad-4c33-8e9c-c7612fbf6c9f",
      • \"url\": "http://example.com?id={{1+1}} - the raw URL",
      • \"link\": "http://example.com?id=2 - the rendered URL, always use this for listing.",
      • \"title\": "Example Website Monitor - manually entered title/description",
      • \"page_title\": "The HTML <title> from the page",
      • \"tags\": [
        • "550e8400-e29b-41d4-a716-446655440000"
        ],
      • \"paused\": false,
      • \"notification_muted\": false,
      • \"method\": "GET",
      • \"fetch_backend\": "html_requests",
      • \"last_checked\": 1640995200,
      • \"last_changed\": 1640995200
      },
    • \"7c9e6b8d-f2a1-4e5c-9d3b-8a7f6e4c2d1a\": {
      • \"uuid\": "7c9e6b8d-f2a1-4e5c-9d3b-8a7f6e4c2d1a",
      • \"url\": "http://example.com?id={{1+1}} - the raw URL",
      • \"link\": "http://example.com?id=2 - the rendered URL, always use this for listing.",
      • \"title\": "News Site Tracker - manually entered title/description",
      • \"page_title\": "The HTML <title> from the page",
      • \"tags\": [
        • "330e8400-e29b-41d4-a716-446655440001"
        ],
      • \"paused\": false,
      • \"notification_muted\": true,
      • \"method\": "GET",
      • \"fetch_backend\": "html_webdriver",
      • \"last_checked\": 1640998800,
      • \"last_changed\": 1640995200
      }
    }
    Create a new watch 
    Create a single web page change monitor (watch). Requires at least url to be set.
    \n
    Every watch can be configured with:
    \n
      \n
    • Processor mode: processor field (restock_diff or text_json_diff - default)
      • \n
    • Notification settings: notification_urls (array), notification_title, notification_body, notification_format, notification_muted
      • \n
    • Tags/Groups: tag (UUID string) or tags (array of UUIDs)
      • \n
    • Check settings: time_between_check, paused, method, fetch_backend
      • \n
    • Advanced options: headers, body, proxy, browser_steps, and more
      • \n
    \n
    Authorizations:
    ApiKeyAuth
    Request Body schema: application/json
    required
    url
    required
    string <uri>   <= 5000 characters 
     
    URL to monitor for changes
    \n
    title
    string or null  <= 5000 characters 
     
    Custom title for the web page change monitor (watch), not to be confused with page_title
    \n
    tag
    string  <= 5000 characters 
     
    Tag UUID to associate with this web page change monitor (watch)
    \n
    tags
    Array of strings
     
    Array of tag UUIDs
    \n
    paused
    boolean
     
    Whether the web page change monitor (watch) is paused
    \n
    notification_muted
    boolean
     
    Whether notifications are muted
    \n
    method
    string
     Enum: "GET" "POST" "DELETE" "PUT" 
     
    HTTP method to use
    \n
    fetch_backend
    string^(system|html_requests|html_webdriver|extra_b...
     Default:  "system"
     
    Backend to use for fetching content. Common values:
    \n
      \n
    • system (default) - Use the system-wide default fetcher
      • \n
    • html_requests - Fast requests-based fetcher
      • \n
    • html_webdriver - Browser-based fetcher (Playwright/Puppeteer)
      • \n
    • extra_browser_* - Custom browser configurations (if configured)
      • \n
    • Plugin-provided fetchers (if installed)
      • \n
    \n
    object
     
    HTTP headers to include in requests
    \n
    body
    string or null  <= 5000 characters 
     
    HTTP request body
    \n
    proxy
    string or null  <= 5000 characters 
     
    Proxy configuration
    \n
    ignore_status_codes
    boolean or null
     
    Ignore HTTP status code errors (boolean or null)
    \n
    webdriver_delay
    integer or null
     
    Delay in seconds for webdriver
    \n
    webdriver_js_execute_code
    string or null  <= 5000 characters 
     
    JavaScript code to execute
    \n
    object
     
    Time intervals between checks. All fields must be non-negative. At least one non-zero value required when not using default settings.
    \n
    time_between_check_use_default
    boolean
     Default:  true
     
    Whether to use global settings for time between checks - defaults to true if not set
    \n
    notification_urls
    Array of strings  <= 100 items [ items  <= 1000 characters  ]
     
    Notification URLs for this web page change monitor (watch). Maximum 100 URLs.
    \n
    notification_title
    string or null  <= 5000 characters 
     
    Custom notification title
    \n
    notification_body
    string or null  <= 5000 characters 
     
    Custom notification body
    \n
    notification_format
    string
     Enum: "text" "html" "htmlcolor" "markdown" "System default" 
     
    Format for notifications
    \n
    track_ldjson_price_data
    boolean or null
     
    Whether to track JSON-LD price data
    \n
    Array of objects  <= 100 items 
     
    Browser automation steps. Maximum 100 steps allowed.
    \n
    processor
    string
     Default:  "text_json_diff"
     Enum: "restock_diff" "text_json_diff" 
     
    Optional processor mode to use for change detection. Defaults to text_json_diff if not specified.
    \n
    include_filters
    Array of strings  <= 100 items [ items  <= 5000 characters  ]
     
    CSS/XPath selectors to extract specific content from the page
    \n
    subtractive_selectors
    Array of strings  <= 100 items [ items  <= 5000 characters  ]
     
    CSS/XPath selectors to remove content from the page
    \n
    ignore_text
    Array of strings  <= 100 items [ items  <= 5000 characters  ]
     
    Text patterns to ignore in change detection
    \n
    trigger_text
    Array of strings  <= 100 items [ items  <= 5000 characters  ]
     
    Text/regex patterns that must be present to trigger a change
    \n
    text_should_not_be_present
    Array of strings  <= 100 items [ items  <= 5000 characters  ]
     
    Text that should NOT be present (triggers alert if found)
    \n
    extract_text
    Array of strings  <= 100 items [ items  <= 5000 characters  ]
     
    Regex patterns to extract specific text after filtering
    \n
    trim_text_whitespace
    boolean
     Default:  false
     
    Strip leading/trailing whitespace from text
    \n
    sort_text_alphabetically
    boolean
     Default:  false
     
    Sort lines alphabetically before comparison
    \n
    remove_duplicate_lines
    boolean
     Default:  false
     
    Remove duplicate lines from content
    \n
    check_unique_lines
    boolean
     Default:  false
     
    Compare against all history for unique lines
    \n
    strip_ignored_lines
    boolean or null
     
    Remove lines matching ignore patterns
    \n
    filter_text_added
    boolean
     Default:  true
     
    Include added text in change detection
    \n
    filter_text_removed
    boolean
     Default:  true
     
    Include removed text in change detection
    \n
    filter_text_replaced
    boolean
     Default:  true
     
    Include replaced text in change detection
    \n
    in_stock_only
    boolean
     Default:  true
     
    Only trigger on in-stock transitions (restock_diff processor)
    \n
    follow_price_changes
    boolean
     Default:  true
     
    Monitor and track price changes (restock_diff processor)
    \n
    price_change_threshold_percent
    number or null
     
    Minimum price change percentage to trigger notification
    \n
    notification_screenshot
    boolean
     Default:  false
     
    Include screenshot in notifications (if supported by notification URL)
    \n
    filter_failure_notification_send
    boolean
     Default:  true
     
    Send notification when filters fail to match content
    \n
    use_page_title_in_list
    boolean or null
     
    Display page title in watch list (null = use system default)
    \n
    history_snapshot_max_length
    integer or null  [ 1 .. 1000 ] 
     
    Maximum number of history snapshots to keep (null = use system default)
    \n
    object
     
    Weekly schedule limiting when checks can run
    \n
    Array of objects  <= 100 items 
     
    Array of condition rules for change detection logic (empty array when not set)
    \n
    conditions_match_logic
    string
     Default:  "ALL"
     Enum: "ALL" "ANY" 
     
    Logic operator - ALL (match all conditions) or ANY (match any condition)
    \n
    Responses
    Development server
    \n
    http://localhost:5000/api/v1/watch
    Production server
    \n
    https://yourdomain.com/api/v1/watch
    Custom server
    \n
    {protocol}://{host}/api/v1/watch
     Request samples 
    • Payload
    • curl
    • Python
    Content type
    application/json
    {
    • \"url\": "https://example.com",
    • \"title\": "Example Site Monitor",
    • \"time_between_check\": {
      • \"hours\": 1
      }
    }
    Get single watch 
    Retrieve web page change monitor (watch) information and set muted/paused status. Returns the FULL Watch JSON.
    \n
    Authorizations:
    ApiKeyAuth
    path Parameters
    uuid
    required
    string <uuid> 
     
    Web page change monitor (watch) unique ID
    \n
    query Parameters
    recheck
    string
     Enum: "1" "true" 
     
    Recheck this web page change monitor (watch)
    \n
    paused
    string
     Enum: "paused" "unpaused" 
     
    Set pause state
    \n
    muted
    string
     Enum: "muted" "unmuted" 
     
    Set mute state
    \n
    Responses
    Development server
    \n
    http://localhost:5000/api/v1/watch/{uuid}
    Production server
    \n
    https://yourdomain.com/api/v1/watch/{uuid}
    Custom server
    \n
    {protocol}://{host}/api/v1/watch/{uuid}
     Request samples 
    • curl
    • Python
    curl -X GET \"http://localhost:5000/api/v1/watch/095be615-a8ad-4c33-8e9c-c7612fbf6c9f\" \\\n  -H \"x-api-key: YOUR_API_KEY\"\n
     Response samples 
    • 200
    • 404
    Content type
    {
    • \"uuid\": "095be615-a8ad-4c33-8e9c-c7612fbf6c9f",
    • \"date_created\": 0,
    • \"url\": "http://example.com",
    • \"title\": "string",
    • \"tag\": "string",
    • \"tags\": [
      • "string"
      ],
    • \"paused\": true,
    • \"notification_muted\": true,
    • \"method\": "GET",
    • \"fetch_backend\": "system",
    • \"headers\": {
      • \"property1\": "string",
      • \"property2\": "string"
      },
    • \"body\": "string",
    • \"proxy\": "string",
    • \"ignore_status_codes\": true,
    • \"webdriver_delay\": 0,
    • \"webdriver_js_execute_code\": "string",
    • \"time_between_check\": {
      • \"weeks\": 52000,
      • \"days\": 365000,
      • \"hours\": 8760000,
      • \"minutes\": 525600000,
      • \"seconds\": 31536000000
      },
    • \"time_between_check_use_default\": true,
    • \"notification_urls\": [
      • "string"
      ],
    • \"notification_title\": "string",
    • \"notification_body\": "string",
    • \"notification_format\": "text",
    • \"track_ldjson_price_data\": true,
    • \"browser_steps\": [
      • {
        • \"operation\": "string",
        • \"selector\": "string",
        • \"optional_value\": "string"
        }
      ],
    • \"processor\": "restock_diff",
    • \"include_filters\": [
      • "string"
      ],
    • \"subtractive_selectors\": [
      • "string"
      ],
    • \"ignore_text\": [
      • "string"
      ],
    • \"trigger_text\": [
      • "string"
      ],
    • \"text_should_not_be_present\": [
      • "string"
      ],
    • \"extract_text\": [
      • "string"
      ],
    • \"trim_text_whitespace\": false,
    • \"sort_text_alphabetically\": false,
    • \"remove_duplicate_lines\": false,
    • \"check_unique_lines\": false,
    • \"strip_ignored_lines\": true,
    • \"filter_text_added\": true,
    • \"filter_text_removed\": true,
    • \"filter_text_replaced\": true,
    • \"in_stock_only\": true,
    • \"follow_price_changes\": true,
    • \"price_change_threshold_percent\": 0,
    • \"has_ldjson_price_data\": true,
    • \"notification_screenshot\": false,
    • \"filter_failure_notification_send\": true,
    • \"use_page_title_in_list\": true,
    • \"history_snapshot_max_length\": 1,
    • \"time_schedule_limit\": {
      • \"enabled\": false,
      • \"monday\": {
        • \"enabled\": true,
        • \"start_time\": "00:00",
        • \"duration\": {
          • \"hours\": "24",
          • \"minutes\": "00"
          }
        },
      • \"tuesday\": {
        • \"enabled\": true,
        • \"start_time\": "00:00",
        • \"duration\": {
          • \"hours\": "24",
          • \"minutes\": "00"
          }
        },
      • \"wednesday\": {
        • \"enabled\": true,
        • \"start_time\": "00:00",
        • \"duration\": {
          • \"hours\": "24",
          • \"minutes\": "00"
          }
        },
      • \"thursday\": {
        • \"enabled\": true,
        • \"start_time\": "00:00",
        • \"duration\": {
          • \"hours\": "24",
          • \"minutes\": "00"
          }
        },
      • \"friday\": {
        • \"enabled\": true,
        • \"start_time\": "00:00",
        • \"duration\": {
          • \"hours\": "24",
          • \"minutes\": "00"
          }
        },
      • \"saturday\": {
        • \"enabled\": true,
        • \"start_time\": "00:00",
        • \"duration\": {
          • \"hours\": "24",
          • \"minutes\": "00"
          }
        },
      • \"sunday\": {
        • \"enabled\": true,
        • \"start_time\": "00:00",
        • \"duration\": {
          • \"hours\": "24",
          • \"minutes\": "00"
          }
        }
      },
    • \"conditions\": [
      • {
        • \"field\": "string",
        • \"operator\": "string",
        • \"value\": "string"
        }
      ],
    • \"conditions_match_logic\": "ALL",
    • \"last_checked\": 0,
    • \"last_changed\": 0,
    • \"last_error\": "string",
    • \"last_viewed\": 0,
    • \"link\": "string",
    • \"page_title\": "string",
    • \"check_count\": 0,
    • \"fetch_time\": 0,
    • \"previous_md5\": "string",
    • \"previous_md5_before_filters\": "string",
    • \"consecutive_filter_failures\": 0,
    • \"last_notification_error\": "string",
    • \"notification_alert_count\": 0,
    • \"content-type\": "string",
    • \"remote_server_reply\": "string",
    • \"browser_steps_last_error_step\": 0,
    • \"viewed\": 0,
    • \"history_n\": 0
    }
    Update watch 
    Update an existing web page change monitor (watch) using JSON. Accepts the same structure as returned in get single watch information.
    \n
    Authorizations:
    ApiKeyAuth
    path Parameters
    uuid
    required
    string <uuid> 
     
    Web page change monitor (watch) unique ID
    \n
    Request Body schema: application/json
    required
    url
    string <uri>   <= 5000 characters 
     
    URL to monitor for changes
    \n
    title
    string or null  <= 5000 characters 
     
    Custom title for the web page change monitor (watch), not to be confused with page_title
    \n
    tag
    string  <= 5000 characters 
     
    Tag UUID to associate with this web page change monitor (watch)
    \n
    tags
    Array of strings
     
    Array of tag UUIDs
    \n
    paused
    boolean
     
    Whether the web page change monitor (watch) is paused
    \n
    notification_muted
    boolean
     
    Whether notifications are muted
    \n
    method
    string
     Enum: "GET" "POST" "DELETE" "PUT" 
     
    HTTP method to use
    \n
    fetch_backend
    string^(system|html_requests|html_webdriver|extra_b...
     Default:  "system"
     
    Backend to use for fetching content. Common values:
    \n
      \n
    • system (default) - Use the system-wide default fetcher
      • \n
    • html_requests - Fast requests-based fetcher
      • \n
    • html_webdriver - Browser-based fetcher (Playwright/Puppeteer)
      • \n
    • extra_browser_* - Custom browser configurations (if configured)
      • \n
    • Plugin-provided fetchers (if installed)
      • \n
    \n
    object
     
    HTTP headers to include in requests
    \n
    body
    string or null  <= 5000 characters 
     
    HTTP request body
    \n
    proxy
    string or null  <= 5000 characters 
     
    Proxy configuration
    \n
    ignore_status_codes
    boolean or null
     
    Ignore HTTP status code errors (boolean or null)
    \n
    webdriver_delay
    integer or null
     
    Delay in seconds for webdriver
    \n
    webdriver_js_execute_code
    string or null  <= 5000 characters 
     
    JavaScript code to execute
    \n
    object
     
    Time intervals between checks. All fields must be non-negative. At least one non-zero value required when not using default settings.
    \n
    time_between_check_use_default
    boolean
     Default:  true
     
    Whether to use global settings for time between checks - defaults to true if not set
    \n
    notification_urls
    Array of strings  <= 100 items [ items  <= 1000 characters  ]
     
    Notification URLs for this web page change monitor (watch). Maximum 100 URLs.
    \n
    notification_title
    string or null  <= 5000 characters 
     
    Custom notification title
    \n
    notification_body
    string or null  <= 5000 characters 
     
    Custom notification body
    \n
    notification_format
    string
     Enum: "text" "html" "htmlcolor" "markdown" "System default" 
     
    Format for notifications
    \n
    track_ldjson_price_data
    boolean or null
     
    Whether to track JSON-LD price data
    \n
    Array of objects  <= 100 items 
     
    Browser automation steps. Maximum 100 steps allowed.
    \n
    processor
    string
     Default:  "text_json_diff"
     Enum: "restock_diff" "text_json_diff" 
     
    Optional processor mode to use for change detection. Defaults to text_json_diff if not specified.
    \n
    include_filters
    Array of strings  <= 100 items [ items  <= 5000 characters  ]
     
    CSS/XPath selectors to extract specific content from the page
    \n
    subtractive_selectors
    Array of strings  <= 100 items [ items  <= 5000 characters  ]
     
    CSS/XPath selectors to remove content from the page
    \n
    ignore_text
    Array of strings  <= 100 items [ items  <= 5000 characters  ]
     
    Text patterns to ignore in change detection
    \n
    trigger_text
    Array of strings  <= 100 items [ items  <= 5000 characters  ]
     
    Text/regex patterns that must be present to trigger a change
    \n
    text_should_not_be_present
    Array of strings  <= 100 items [ items  <= 5000 characters  ]
     
    Text that should NOT be present (triggers alert if found)
    \n
    extract_text
    Array of strings  <= 100 items [ items  <= 5000 characters  ]
     
    Regex patterns to extract specific text after filtering
    \n
    trim_text_whitespace
    boolean
     Default:  false
     
    Strip leading/trailing whitespace from text
    \n
    sort_text_alphabetically
    boolean
     Default:  false
     
    Sort lines alphabetically before comparison
    \n
    remove_duplicate_lines
    boolean
     Default:  false
     
    Remove duplicate lines from content
    \n
    check_unique_lines
    boolean
     Default:  false
     
    Compare against all history for unique lines
    \n
    strip_ignored_lines
    boolean or null
     
    Remove lines matching ignore patterns
    \n
    filter_text_added
    boolean
     Default:  true
     
    Include added text in change detection
    \n
    filter_text_removed
    boolean
     Default:  true
     
    Include removed text in change detection
    \n
    filter_text_replaced
    boolean
     Default:  true
     
    Include replaced text in change detection
    \n
    in_stock_only
    boolean
     Default:  true
     
    Only trigger on in-stock transitions (restock_diff processor)
    \n
    follow_price_changes
    boolean
     Default:  true
     
    Monitor and track price changes (restock_diff processor)
    \n
    price_change_threshold_percent
    number or null
     
    Minimum price change percentage to trigger notification
    \n
    notification_screenshot
    boolean
     Default:  false
     
    Include screenshot in notifications (if supported by notification URL)
    \n
    filter_failure_notification_send
    boolean
     Default:  true
     
    Send notification when filters fail to match content
    \n
    use_page_title_in_list
    boolean or null
     
    Display page title in watch list (null = use system default)
    \n
    history_snapshot_max_length
    integer or null  [ 1 .. 1000 ] 
     
    Maximum number of history snapshots to keep (null = use system default)
    \n
    object
     
    Weekly schedule limiting when checks can run
    \n
    Array of objects  <= 100 items 
     
    Array of condition rules for change detection logic (empty array when not set)
    \n
    conditions_match_logic
    string
     Default:  "ALL"
     Enum: "ALL" "ANY" 
     
    Logic operator - ALL (match all conditions) or ANY (match any condition)
    \n
    last_viewed
    integer  >= 0 
     
    Unix timestamp in seconds of the last time the watch was viewed. Setting it to a value higher than last_changed in the "Update watch" endpoint marks the watch as viewed.
    \n
    Responses
    Development server
    \n
    http://localhost:5000/api/v1/watch/{uuid}
    Production server
    \n
    https://yourdomain.com/api/v1/watch/{uuid}
    Custom server
    \n
    {protocol}://{host}/api/v1/watch/{uuid}
     Request samples 
    • Payload
    • curl
    • Python
    Content type
    application/json
    {
    • \"url\": "http://example.com",
    • \"title\": "string",
    • \"tag\": "string",
    • \"tags\": [
      • "string"
      ],
    • \"paused\": true,
    • \"notification_muted\": true,
    • \"method\": "GET",
    • \"fetch_backend\": "system",
    • \"headers\": {
      • \"property1\": "string",
      • \"property2\": "string"
      },
    • \"body\": "string",
    • \"proxy\": "string",
    • \"ignore_status_codes\": true,
    • \"webdriver_delay\": 0,
    • \"webdriver_js_execute_code\": "string",
    • \"time_between_check\": {
      • \"weeks\": 52000,
      • \"days\": 365000,
      • \"hours\": 8760000,
      • \"minutes\": 525600000,
      • \"seconds\": 31536000000
      },
    • \"time_between_check_use_default\": true,
    • \"notification_urls\": [
      • "string"
      ],
    • \"notification_title\": "string",
    • \"notification_body\": "string",
    • \"notification_format\": "text",
    • \"track_ldjson_price_data\": true,
    • \"browser_steps\": [
      • {
        • \"operation\": "string",
        • \"selector\": "string",
        • \"optional_value\": "string"
        }
      ],
    • \"processor\": "restock_diff",
    • \"include_filters\": [
      • "string"
      ],
    • \"subtractive_selectors\": [
      • "string"
      ],
    • \"ignore_text\": [
      • "string"
      ],
    • \"trigger_text\": [
      • "string"
      ],
    • \"text_should_not_be_present\": [
      • "string"
      ],
    • \"extract_text\": [
      • "string"
      ],
    • \"trim_text_whitespace\": false,
    • \"sort_text_alphabetically\": false,
    • \"remove_duplicate_lines\": false,
    • \"check_unique_lines\": false,
    • \"strip_ignored_lines\": true,
    • \"filter_text_added\": true,
    • \"filter_text_removed\": true,
    • \"filter_text_replaced\": true,
    • \"in_stock_only\": true,
    • \"follow_price_changes\": true,
    • \"price_change_threshold_percent\": 0,
    • \"notification_screenshot\": false,
    • \"filter_failure_notification_send\": true,
    • \"use_page_title_in_list\": true,
    • \"history_snapshot_max_length\": 1,
    • \"time_schedule_limit\": {
      • \"enabled\": false,
      • \"monday\": {
        • \"enabled\": true,
        • \"start_time\": "00:00",
        • \"duration\": {
          • \"hours\": "24",
          • \"minutes\": "00"
          }
        },
      • \"tuesday\": {
        • \"enabled\": true,
        • \"start_time\": "00:00",
        • \"duration\": {
          • \"hours\": "24",
          • \"minutes\": "00"
          }
        },
      • \"wednesday\": {
        • \"enabled\": true,
        • \"start_time\": "00:00",
        • \"duration\": {
          • \"hours\": "24",
          • \"minutes\": "00"
          }
        },
      • \"thursday\": {
        • \"enabled\": true,
        • \"start_time\": "00:00",
        • \"duration\": {
          • \"hours\": "24",
          • \"minutes\": "00"
          }
        },
      • \"friday\": {
        • \"enabled\": true,
        • \"start_time\": "00:00",
        • \"duration\": {
          • \"hours\": "24",
          • \"minutes\": "00"
          }
        },
      • \"saturday\": {
        • \"enabled\": true,
        • \"start_time\": "00:00",
        • \"duration\": {
          • \"hours\": "24",
          • \"minutes\": "00"
          }
        },
      • \"sunday\": {
        • \"enabled\": true,
        • \"start_time\": "00:00",
        • \"duration\": {
          • \"hours\": "24",
          • \"minutes\": "00"
          }
        }
      },
    • \"conditions\": [
      • {
        • \"field\": "string",
        • \"operator\": "string",
        • \"value\": "string"
        }
      ],
    • \"conditions_match_logic\": "ALL",
    • \"last_viewed\": 0
    }
    Delete watch 
    Delete a web page change monitor (watch) and all related history
    \n
    Authorizations:
    ApiKeyAuth
    path Parameters
    uuid
    required
    string <uuid> 
     
    Web page change monitor (watch) unique ID
    \n
    Responses
    Development server
    \n
    http://localhost:5000/api/v1/watch/{uuid}
    Production server
    \n
    https://yourdomain.com/api/v1/watch/{uuid}
    Custom server
    \n
    {protocol}://{host}/api/v1/watch/{uuid}
     Request samples 
    • curl
    • Python
    curl -X DELETE \"http://localhost:5000/api/v1/watch/095be615-a8ad-4c33-8e9c-c7612fbf6c9f\" \\\n  -H \"x-api-key: YOUR_API_KEY\"\n
    Watch History
    Get a list of timestamps of all changes detected for a watch.
    \n
    Get watch history 
    Get a list of all historical snapshots available for a web page change monitor (watch), use the key timestamp\nas the query argument for fetching a single watch history snapshot.
    \n
    Authorizations:
    ApiKeyAuth
    path Parameters
    uuid
    required
    string <uuid> 
     
    Web page change monitor (watch) unique ID
    \n
    Responses
    Development server
    \n
    http://localhost:5000/api/v1/watch/{uuid}/history
    Production server
    \n
    https://yourdomain.com/api/v1/watch/{uuid}/history
    Custom server
    \n
    {protocol}://{host}/api/v1/watch/{uuid}/history
     Request samples 
    • curl
    • Python
    curl -X GET \"http://localhost:5000/api/v1/watch/095be615-a8ad-4c33-8e9c-c7612fbf6c9f/history\" \\\n  -H \"x-api-key: YOUR_API_KEY\"\n
     Response samples 
    • 200
    Content type
    application/json
    {
    • \"1640995200\": "/path/to/snapshot1.txt",
    • \"1640998800\": "/path/to/snapshot2.txt"
    }
    Get the difference between two snapshots 
    Generate a difference (comparison) between two historical snapshots of a web page change monitor (watch).
    \n
    This endpoint compares content between two points in time and returns the differences in your chosen format.\nPerfect for reviewing what changed between specific versions or comparing recent changes.
    \n
    Timestamp Keywords:
    \n
      \n
    • Use 'latest' for the most recent snapshot (to_timestamp)
      • \n
    • Use 'previous' for the second-most-recent snapshot (from_timestamp)
      • \n
    • Or use specific Unix timestamps from the watch history
      • \n
    \n
    Format Options:
    \n
      \n
    • text (default): Plain text with (removed) and (added) prefixes
      • \n
    • html: HTML format with (removed) and (added) text
      • \n
    • htmlcolor: Rich HTML with colored highlights (green for additions, red for deletions)
      • \n
    \n
    Word-Level Diffing:
    \n
      \n
    • Enable word-level granularity with word_diff=true for detailed inline comparisons
      • \n
    • Disable with word_diff=false for line-level comparisons only (default false/off, line-level mode by default)
      • \n
    \n
    Raw Diff Output:
    \n
      \n
    • Use no_markup=true to get raw diff content without any formatting applied
      • \n
    • Returns content with placeholders for opening/closing tags of changes
      • \n
    • Allows you to implement your own custom colorisation or formatting
      • \n
    • Skips all HTML color application and service tweaks (added text, html color tags, etc)
      • \n
    \n
    Authorizations:
    ApiKeyAuth
    path Parameters
    uuid
    required
    string <uuid> 
     
    Web page change monitor (watch) unique ID
    \n
    required
    integer or string
     
     Example:  previous
    Starting snapshot timestamp, 'previous' for second-most-recent, or specific Unix timestamp
    \n
    required
    integer or string
     
     Example:  latest
    Ending snapshot timestamp, 'latest' for most recent, or specific Unix timestamp
    \n
    query Parameters
    format
    string
     Default:  "text"
     Enum: "text" "html" "htmlcolor" "markdown" 
     
    Output format for the diff:
    \n
      \n
    • text (default): Plain text with (removed) and (added) prefixes
      • \n
    • html: Basic HTML format
      • \n
    • htmlcolor: Rich HTML with colored backgrounds (red for deletions, green for additions)
      • \n
    • markdown: Markdown format with HTML rendering
      • \n
    \n
    word_diff
    string
     Default:  "false"
     Enum: "true" "false" "1" "0" "yes" "no" "on" "off" 
     
    Enable word-level diffing for more granular comparisons.\nWhen enabled, changes are highlighted at the word level rather than line level.\nDefault is false (line-level mode).\nAccepts: true, false, 1, 0, yes, no, on, off
    \n
    no_markup
    string
     Default:  "false"
     Enum: "true" "false" "1" "0" "yes" "no" "on" "off" 
     
    When set to true, returns the raw diff content without any markup formatting.\nThe content will include placeholders for opening/closing tags of the changes,\nallowing you to implement your own custom colorisation or formatting.\nThis skips all HTML color application and service tweaks.\nAccepts: true, false, 1, 0, yes, no, on, off
    \n
    type
    string
     Default:  "diffLines"
     Enum: "diffLines" "diffWords" 
     
    Diff granularity type:
    \n
      \n
    • diffLines (default): Line-level comparison, showing which lines changed
      • \n
    • diffWords: Word-level comparison, showing which words changed within lines
      • \n
    \n
    This parameter is an alternative to word_diff for better alignment with the UI.\nIf both are specified, type=diffWords will enable word-level diffing.
    \n
    changesOnly
    string
     Default:  "true"
     Enum: "true" "false" "1" "0" "yes" "no" "on" "off" 
     
    When enabled, only show lines/content that changed (no surrounding context).\nWhen disabled, include unchanged lines for context around changes.\nAccepts: true, false, 1, 0, yes, no, on, off
    \n
    ignoreWhitespace
    string
     Default:  "false"
     Enum: "true" "false" "1" "0" "yes" "no" "on" "off" 
     
    When enabled, ignore whitespace-only changes (spaces, tabs, newlines).\nUseful for focusing on content changes and ignoring formatting differences.\nAccepts: true, false, 1, 0, yes, no, on, off
    \n
    removed
    string
     Default:  "true"
     Enum: "true" "false" "1" "0" "yes" "no" "on" "off" 
     
    Include removed/deleted content in the diff output.\nWhen disabled, content that was deleted will not appear in the diff.\nAccepts: true, false, 1, 0, yes, no, on, off
    \n
    added
    string
     Default:  "true"
     Enum: "true" "false" "1" "0" "yes" "no" "on" "off" 
     
    Include added/new content in the diff output.\nWhen disabled, content that was added will not appear in the diff.\nAccepts: true, false, 1, 0, yes, no, on, off
    \n
    replaced
    string
     Default:  "true"
     Enum: "true" "false" "1" "0" "yes" "no" "on" "off" 
     
    Include replaced/modified content in the diff output.\nWhen disabled, content that was modified (changed from one value to another) will not appear in the diff.\nAccepts: true, false, 1, 0, yes, no, on, off
    \n
    Responses
    Development server
    \n
    http://localhost:5000/api/v1/watch/{uuid}/difference/{from_timestamp}/{to_timestamp}
    Production server
    \n
    https://yourdomain.com/api/v1/watch/{uuid}/difference/{from_timestamp}/{to_timestamp}
    Custom server
    \n
    {protocol}://{host}/api/v1/watch/{uuid}/difference/{from_timestamp}/{to_timestamp}
     Request samples 
    • curl
    • Python
    # Compare previous snapshot to latest with colored HTML\ncurl -X GET \"http://localhost:5000/api/v1/watch/095be615-a8ad-4c33-8e9c-c7612fbf6c9f/difference/previous/latest?format=htmlcolor\" \\\n  -H \"x-api-key: YOUR_API_KEY\"\n\n# Compare two specific timestamps in plain text with word-level diff\ncurl -X GET \"http://localhost:5000/api/v1/watch/095be615-a8ad-4c33-8e9c-c7612fbf6c9f/difference/1640995200/1640998800?format=text&word_diff=true\" \\\n  -H \"x-api-key: YOUR_API_KEY\"\n\n# Show only additions (hide removed/replaced content), ignore whitespace\ncurl -X GET \"http://localhost:5000/api/v1/watch/095be615-a8ad-4c33-8e9c-c7612fbf6c9f/difference/previous/latest?format=htmlcolor&removed=false&replaced=false&ignoreWhitespace=true\" \\\n  -H \"x-api-key: YOUR_API_KEY\"\n
    Snapshots
    Retrieve individual text snapshot of monitored content according to the timestamp. The text snapshot is the HTML\nto Text at page check time. 
    \n
    Set the query argument html to any value to retrieve the last HTML fetched, the system only keeps the last two \n(2) HTML files fetched.
    \n
    Use the Watch History API endpoint to get a list of timestamps to pass to this query.
    \n
    Get single snapshot 
    Get single snapshot from web page change monitor (watch). Use 'latest' for the most recent snapshot.\nUse the Watch History API to get a list of timestamps to pass.
    \n
    Authorizations:
    ApiKeyAuth
    path Parameters
    uuid
    required
    string <uuid> 
     
    Web page change monitor (watch) unique ID
    \n
    required
    integer or string
     
    Snapshot timestamp or 'latest'
    \n
    query Parameters
    html
    string
     Value: "1" 
     
    Set to 1 to return the last HTML
    \n
    Responses
    Development server
    \n
    http://localhost:5000/api/v1/watch/{uuid}/history/{timestamp}
    Production server
    \n
    https://yourdomain.com/api/v1/watch/{uuid}/history/{timestamp}
    Custom server
    \n
    {protocol}://{host}/api/v1/watch/{uuid}/history/{timestamp}
     Request samples 
    • curl
    • Python
    curl -X GET \"http://localhost:5000/api/v1/watch/095be615-a8ad-4c33-8e9c-c7612fbf6c9f/history/latest\" \\\n  -H \"x-api-key: YOUR_API_KEY\"\n
    Favicon
    Retrieve favicon images associated with monitored web pages. These are used in the dashboard interface \nto visually identify different watches in your monitoring list.
    \n
    Get watch favicon 
    Get the favicon for a web page change monitor (watch) as displayed in the watch overview list.
    \n
    Authorizations:
    ApiKeyAuth
    path Parameters
    uuid
    required
    string <uuid> 
     
    Web page change monitor (watch) unique ID
    \n
    Responses
    Development server
    \n
    http://localhost:5000/api/v1/watch/{uuid}/favicon
    Production server
    \n
    https://yourdomain.com/api/v1/watch/{uuid}/favicon
    Custom server
    \n
    {protocol}://{host}/api/v1/watch/{uuid}/favicon
     Request samples 
    • curl
    • Python
    curl -X GET \"http://localhost:5000/api/v1/watch/095be615-a8ad-4c33-8e9c-c7612fbf6c9f/favicon\" \\\n  -H \"x-api-key: YOUR_API_KEY\" \\\n  --output favicon.ico\n
    Group / Tag Management
    Organize your watches using tags and groups. Tags (also known as Groups) allow you to categorize monitors, set group-wide \nnotification preferences, and perform bulk operations like mass rechecking or status changes across \nmultiple related watches.
    \n
    List all tags 
    Return list of available tags/groups
    \n
    Authorizations:
    ApiKeyAuth
    Responses
    Development server
    \n
    http://localhost:5000/api/v1/tags
    Production server
    \n
    https://yourdomain.com/api/v1/tags
    Custom server
    \n
    {protocol}://{host}/api/v1/tags
     Request samples 
    • curl
    • Python
    curl -X GET \"http://localhost:5000/api/v1/tags\" \\\n  -H \"x-api-key: YOUR_API_KEY\"\n
     Response samples 
    • 200
    Content type
    application/json
    {
    • \"550e8400-e29b-41d4-a716-446655440000\": {
      • \"uuid\": "550e8400-e29b-41d4-a716-446655440000",
      • \"title\": "Production Sites",
      • \"notification_urls\": [
        • "mailto:admin@example.com"
        ],
      • \"notification_muted\": false
      },
    • \"330e8400-e29b-41d4-a716-446655440001\": {
      • \"uuid\": "330e8400-e29b-41d4-a716-446655440001",
      • \"title\": "News Sources",
      • \"notification_urls\": [
        • "discord://webhook_id/webhook_token"
        ],
      • \"notification_muted\": false
      }
    }
    Create tag 
    Create a single tag/group
    \n
    Authorizations:
    ApiKeyAuth
    Request Body schema: application/json
    required
    url
    string <uri>   <= 5000 characters 
     
    URL to monitor for changes
    \n
    title
    required
    string or null  <= 5000 characters 
     
    Custom title for the web page change monitor (watch), not to be confused with page_title
    \n
    tag
    string  <= 5000 characters 
     
    Tag UUID to associate with this web page change monitor (watch)
    \n
    tags
    Array of strings
     
    Array of tag UUIDs
    \n
    paused
    boolean
     
    Whether the web page change monitor (watch) is paused
    \n
    notification_muted
    boolean
     
    Whether notifications are muted
    \n
    method
    string
     Enum: "GET" "POST" "DELETE" "PUT" 
     
    HTTP method to use
    \n
    fetch_backend
    string^(system|html_requests|html_webdriver|extra_b...
     Default:  "system"
     
    Backend to use for fetching content. Common values:
    \n
      \n
    • system (default) - Use the system-wide default fetcher
      • \n
    • html_requests - Fast requests-based fetcher
      • \n
    • html_webdriver - Browser-based fetcher (Playwright/Puppeteer)
      • \n
    • extra_browser_* - Custom browser configurations (if configured)
      • \n
    • Plugin-provided fetchers (if installed)
      • \n
    \n
    object
     
    HTTP headers to include in requests
    \n
    body
    string or null  <= 5000 characters 
     
    HTTP request body
    \n
    proxy
    string or null  <= 5000 characters 
     
    Proxy configuration
    \n
    ignore_status_codes
    boolean or null
     
    Ignore HTTP status code errors (boolean or null)
    \n
    webdriver_delay
    integer or null
     
    Delay in seconds for webdriver
    \n
    webdriver_js_execute_code
    string or null  <= 5000 characters 
     
    JavaScript code to execute
    \n
    object
     
    Time intervals between checks. All fields must be non-negative. At least one non-zero value required when not using default settings.
    \n
    time_between_check_use_default
    boolean
     Default:  true
     
    Whether to use global settings for time between checks - defaults to true if not set
    \n
    notification_urls
    Array of strings  <= 100 items [ items  <= 1000 characters  ]
     
    Notification URLs for this web page change monitor (watch). Maximum 100 URLs.
    \n
    notification_title
    string or null  <= 5000 characters 
     
    Custom notification title
    \n
    notification_body
    string or null  <= 5000 characters 
     
    Custom notification body
    \n
    notification_format
    string
     Enum: "text" "html" "htmlcolor" "markdown" "System default" 
     
    Format for notifications
    \n
    track_ldjson_price_data
    boolean or null
     
    Whether to track JSON-LD price data
    \n
    Array of objects  <= 100 items 
     
    Browser automation steps. Maximum 100 steps allowed.
    \n
    processor
    string
     Default:  "text_json_diff"
     Enum: "restock_diff" "text_json_diff" 
     
    Optional processor mode to use for change detection. Defaults to text_json_diff if not specified.
    \n
    include_filters
    Array of strings  <= 100 items [ items  <= 5000 characters  ]
     
    CSS/XPath selectors to extract specific content from the page
    \n
    subtractive_selectors
    Array of strings  <= 100 items [ items  <= 5000 characters  ]
     
    CSS/XPath selectors to remove content from the page
    \n
    ignore_text
    Array of strings  <= 100 items [ items  <= 5000 characters  ]
     
    Text patterns to ignore in change detection
    \n
    trigger_text
    Array of strings  <= 100 items [ items  <= 5000 characters  ]
     
    Text/regex patterns that must be present to trigger a change
    \n
    text_should_not_be_present
    Array of strings  <= 100 items [ items  <= 5000 characters  ]
     
    Text that should NOT be present (triggers alert if found)
    \n
    extract_text
    Array of strings  <= 100 items [ items  <= 5000 characters  ]
     
    Regex patterns to extract specific text after filtering
    \n
    trim_text_whitespace
    boolean
     Default:  false
     
    Strip leading/trailing whitespace from text
    \n
    sort_text_alphabetically
    boolean
     Default:  false
     
    Sort lines alphabetically before comparison
    \n
    remove_duplicate_lines
    boolean
     Default:  false
     
    Remove duplicate lines from content
    \n
    check_unique_lines
    boolean
     Default:  false
     
    Compare against all history for unique lines
    \n
    strip_ignored_lines
    boolean or null
     
    Remove lines matching ignore patterns
    \n
    filter_text_added
    boolean
     Default:  true
     
    Include added text in change detection
    \n
    filter_text_removed
    boolean
     Default:  true
     
    Include removed text in change detection
    \n
    filter_text_replaced
    boolean
     Default:  true
     
    Include replaced text in change detection
    \n
    in_stock_only
    boolean
     Default:  true
     
    Only trigger on in-stock transitions (restock_diff processor)
    \n
    follow_price_changes
    boolean
     Default:  true
     
    Monitor and track price changes (restock_diff processor)
    \n
    price_change_threshold_percent
    number or null
     
    Minimum price change percentage to trigger notification
    \n
    notification_screenshot
    boolean
     Default:  false
     
    Include screenshot in notifications (if supported by notification URL)
    \n
    filter_failure_notification_send
    boolean
     Default:  true
     
    Send notification when filters fail to match content
    \n
    use_page_title_in_list
    boolean or null
     
    Display page title in watch list (null = use system default)
    \n
    history_snapshot_max_length
    integer or null  [ 1 .. 1000 ] 
     
    Maximum number of history snapshots to keep (null = use system default)
    \n
    object
     
    Weekly schedule limiting when checks can run
    \n
    Array of objects  <= 100 items 
     
    Array of condition rules for change detection logic (empty array when not set)
    \n
    conditions_match_logic
    string
     Default:  "ALL"
     Enum: "ALL" "ANY" 
     
    Logic operator - ALL (match all conditions) or ANY (match any condition)
    \n
    overrides_watch
    boolean or null
     
    Whether this tag's settings override watch settings for all watches in this tag/group.
    \n
      \n
    • true: Tag settings override watch settings
      • \n
    • false: Tag settings do not override (watches use their own settings)
      • \n
    • null: Not decided yet / inherit default behavior
      • \n
    \n
    Responses
    Development server
    \n
    http://localhost:5000/api/v1/tag
    Production server
    \n
    https://yourdomain.com/api/v1/tag
    Custom server
    \n
    {protocol}://{host}/api/v1/tag
     Request samples 
    • Payload
    • curl
    • Python
    Content type
    application/json
    {
    • \"title\": "Important Sites"
    }
     Response samples 
    • 201
    Content type
    application/json
    {
    • \"uuid\": "095be615-a8ad-4c33-8e9c-c7612fbf6c9f"
    }
    Get single tag 
    Retrieve tag information, set notification_muted status, recheck all web page change monitors (watches) in tag.
    \n
    Authorizations:
    ApiKeyAuth
    path Parameters
    uuid
    required
    string <uuid> 
     
    Tag unique ID
    \n
    query Parameters
    muted
    string
     Enum: "muted" "unmuted" 
     
    Set mute state
    \n
    recheck
    string
     Value: "true" 
     
    Queue all web page change monitors (watches) with this tag for recheck
    \n
    Responses
    Development server
    \n
    http://localhost:5000/api/v1/tag/{uuid}
    Production server
    \n
    https://yourdomain.com/api/v1/tag/{uuid}
    Custom server
    \n
    {protocol}://{host}/api/v1/tag/{uuid}
     Request samples 
    • curl
    • Python
    curl -X GET \"http://localhost:5000/api/v1/tag/550e8400-e29b-41d4-a716-446655440000\" \\\n  -H \"x-api-key: YOUR_API_KEY\"\n
     Response samples 
    • 200
    Content type
    {
    • \"uuid\": "095be615-a8ad-4c33-8e9c-c7612fbf6c9f",
    • \"date_created\": 0,
    • \"url\": "http://example.com",
    • \"title\": "string",
    • \"tag\": "string",
    • \"tags\": [
      • "string"
      ],
    • \"paused\": true,
    • \"notification_muted\": true,
    • \"method\": "GET",
    • \"fetch_backend\": "system",
    • \"headers\": {
      • \"property1\": "string",
      • \"property2\": "string"
      },
    • \"body\": "string",
    • \"proxy\": "string",
    • \"ignore_status_codes\": true,
    • \"webdriver_delay\": 0,
    • \"webdriver_js_execute_code\": "string",
    • \"time_between_check\": {
      • \"weeks\": 52000,
      • \"days\": 365000,
      • \"hours\": 8760000,
      • \"minutes\": 525600000,
      • \"seconds\": 31536000000
      },
    • \"time_between_check_use_default\": true,
    • \"notification_urls\": [
      • "string"
      ],
    • \"notification_title\": "string",
    • \"notification_body\": "string",
    • \"notification_format\": "text",
    • \"track_ldjson_price_data\": true,
    • \"browser_steps\": [
      • {
        • \"operation\": "string",
        • \"selector\": "string",
        • \"optional_value\": "string"
        }
      ],
    • \"processor\": "restock_diff",
    • \"include_filters\": [
      • "string"
      ],
    • \"subtractive_selectors\": [
      • "string"
      ],
    • \"ignore_text\": [
      • "string"
      ],
    • \"trigger_text\": [
      • "string"
      ],
    • \"text_should_not_be_present\": [
      • "string"
      ],
    • \"extract_text\": [
      • "string"
      ],
    • \"trim_text_whitespace\": false,
    • \"sort_text_alphabetically\": false,
    • \"remove_duplicate_lines\": false,
    • \"check_unique_lines\": false,
    • \"strip_ignored_lines\": true,
    • \"filter_text_added\": true,
    • \"filter_text_removed\": true,
    • \"filter_text_replaced\": true,
    • \"in_stock_only\": true,
    • \"follow_price_changes\": true,
    • \"price_change_threshold_percent\": 0,
    • \"has_ldjson_price_data\": true,
    • \"notification_screenshot\": false,
    • \"filter_failure_notification_send\": true,
    • \"use_page_title_in_list\": true,
    • \"history_snapshot_max_length\": 1,
    • \"time_schedule_limit\": {
      • \"enabled\": false,
      • \"monday\": {
        • \"enabled\": true,
        • \"start_time\": "00:00",
        • \"duration\": {
          • \"hours\": "24",
          • \"minutes\": "00"
          }
        },
      • \"tuesday\": {
        • \"enabled\": true,
        • \"start_time\": "00:00",
        • \"duration\": {
          • \"hours\": "24",
          • \"minutes\": "00"
          }
        },
      • \"wednesday\": {
        • \"enabled\": true,
        • \"start_time\": "00:00",
        • \"duration\": {
          • \"hours\": "24",
          • \"minutes\": "00"
          }
        },
      • \"thursday\": {
        • \"enabled\": true,
        • \"start_time\": "00:00",
        • \"duration\": {
          • \"hours\": "24",
          • \"minutes\": "00"
          }
        },
      • \"friday\": {
        • \"enabled\": true,
        • \"start_time\": "00:00",
        • \"duration\": {
          • \"hours\": "24",
          • \"minutes\": "00"
          }
        },
      • \"saturday\": {
        • \"enabled\": true,
        • \"start_time\": "00:00",
        • \"duration\": {
          • \"hours\": "24",
          • \"minutes\": "00"
          }
        },
      • \"sunday\": {
        • \"enabled\": true,
        • \"start_time\": "00:00",
        • \"duration\": {
          • \"hours\": "24",
          • \"minutes\": "00"
          }
        }
      },
    • \"conditions\": [
      • {
        • \"field\": "string",
        • \"operator\": "string",
        • \"value\": "string"
        }
      ],
    • \"conditions_match_logic\": "ALL",
    • \"overrides_watch\": true
    }
    Update tag 
    Update an existing tag using JSON
    \n
    Authorizations:
    ApiKeyAuth
    path Parameters
    uuid
    required
    string <uuid> 
     
    Tag unique ID
    \n
    Request Body schema: application/json
    required
    url
    string <uri>   <= 5000 characters 
     
    URL to monitor for changes
    \n
    title
    string or null  <= 5000 characters 
     
    Custom title for the web page change monitor (watch), not to be confused with page_title
    \n
    tag
    string  <= 5000 characters 
     
    Tag UUID to associate with this web page change monitor (watch)
    \n
    tags
    Array of strings
     
    Array of tag UUIDs
    \n
    paused
    boolean
     
    Whether the web page change monitor (watch) is paused
    \n
    notification_muted
    boolean
     
    Whether notifications are muted
    \n
    method
    string
     Enum: "GET" "POST" "DELETE" "PUT" 
     
    HTTP method to use
    \n
    fetch_backend
    string^(system|html_requests|html_webdriver|extra_b...
     Default:  "system"
     
    Backend to use for fetching content. Common values:
    \n
      \n
    • system (default) - Use the system-wide default fetcher
      • \n
    • html_requests - Fast requests-based fetcher
      • \n
    • html_webdriver - Browser-based fetcher (Playwright/Puppeteer)
      • \n
    • extra_browser_* - Custom browser configurations (if configured)
      • \n
    • Plugin-provided fetchers (if installed)
      • \n
    \n
    object
     
    HTTP headers to include in requests
    \n
    body
    string or null  <= 5000 characters 
     
    HTTP request body
    \n
    proxy
    string or null  <= 5000 characters 
     
    Proxy configuration
    \n
    ignore_status_codes
    boolean or null
     
    Ignore HTTP status code errors (boolean or null)
    \n
    webdriver_delay
    integer or null
     
    Delay in seconds for webdriver
    \n
    webdriver_js_execute_code
    string or null  <= 5000 characters 
     
    JavaScript code to execute
    \n
    object
     
    Time intervals between checks. All fields must be non-negative. At least one non-zero value required when not using default settings.
    \n
    time_between_check_use_default
    boolean
     Default:  true
     
    Whether to use global settings for time between checks - defaults to true if not set
    \n
    notification_urls
    Array of strings  <= 100 items [ items  <= 1000 characters  ]
     
    Notification URLs for this web page change monitor (watch). Maximum 100 URLs.
    \n
    notification_title
    string or null  <= 5000 characters 
     
    Custom notification title
    \n
    notification_body
    string or null  <= 5000 characters 
     
    Custom notification body
    \n
    notification_format
    string
     Enum: "text" "html" "htmlcolor" "markdown" "System default" 
     
    Format for notifications
    \n
    track_ldjson_price_data
    boolean or null
     
    Whether to track JSON-LD price data
    \n
    Array of objects  <= 100 items 
     
    Browser automation steps. Maximum 100 steps allowed.
    \n
    processor
    string
     Default:  "text_json_diff"
     Enum: "restock_diff" "text_json_diff" 
     
    Optional processor mode to use for change detection. Defaults to text_json_diff if not specified.
    \n
    include_filters
    Array of strings  <= 100 items [ items  <= 5000 characters  ]
     
    CSS/XPath selectors to extract specific content from the page
    \n
    subtractive_selectors
    Array of strings  <= 100 items [ items  <= 5000 characters  ]
     
    CSS/XPath selectors to remove content from the page
    \n
    ignore_text
    Array of strings  <= 100 items [ items  <= 5000 characters  ]
     
    Text patterns to ignore in change detection
    \n
    trigger_text
    Array of strings  <= 100 items [ items  <= 5000 characters  ]
     
    Text/regex patterns that must be present to trigger a change
    \n
    text_should_not_be_present
    Array of strings  <= 100 items [ items  <= 5000 characters  ]
     
    Text that should NOT be present (triggers alert if found)
    \n
    extract_text
    Array of strings  <= 100 items [ items  <= 5000 characters  ]
     
    Regex patterns to extract specific text after filtering
    \n
    trim_text_whitespace
    boolean
     Default:  false
     
    Strip leading/trailing whitespace from text
    \n
    sort_text_alphabetically
    boolean
     Default:  false
     
    Sort lines alphabetically before comparison
    \n
    remove_duplicate_lines
    boolean
     Default:  false
     
    Remove duplicate lines from content
    \n
    check_unique_lines
    boolean
     Default:  false
     
    Compare against all history for unique lines
    \n
    strip_ignored_lines
    boolean or null
     
    Remove lines matching ignore patterns
    \n
    filter_text_added
    boolean
     Default:  true
     
    Include added text in change detection
    \n
    filter_text_removed
    boolean
     Default:  true
     
    Include removed text in change detection
    \n
    filter_text_replaced
    boolean
     Default:  true
     
    Include replaced text in change detection
    \n
    in_stock_only
    boolean
     Default:  true
     
    Only trigger on in-stock transitions (restock_diff processor)
    \n
    follow_price_changes
    boolean
     Default:  true
     
    Monitor and track price changes (restock_diff processor)
    \n
    price_change_threshold_percent
    number or null
     
    Minimum price change percentage to trigger notification
    \n
    notification_screenshot
    boolean
     Default:  false
     
    Include screenshot in notifications (if supported by notification URL)
    \n
    filter_failure_notification_send
    boolean
     Default:  true
     
    Send notification when filters fail to match content
    \n
    use_page_title_in_list
    boolean or null
     
    Display page title in watch list (null = use system default)
    \n
    history_snapshot_max_length
    integer or null  [ 1 .. 1000 ] 
     
    Maximum number of history snapshots to keep (null = use system default)
    \n
    object
     
    Weekly schedule limiting when checks can run
    \n
    Array of objects  <= 100 items 
     
    Array of condition rules for change detection logic (empty array when not set)
    \n
    conditions_match_logic
    string
     Default:  "ALL"
     Enum: "ALL" "ANY" 
     
    Logic operator - ALL (match all conditions) or ANY (match any condition)
    \n
    overrides_watch
    boolean or null
     
    Whether this tag's settings override watch settings for all watches in this tag/group.
    \n
      \n
    • true: Tag settings override watch settings
      • \n
    • false: Tag settings do not override (watches use their own settings)
      • \n
    • null: Not decided yet / inherit default behavior
      • \n
    \n
    Responses
    Development server
    \n
    http://localhost:5000/api/v1/tag/{uuid}
    Production server
    \n
    https://yourdomain.com/api/v1/tag/{uuid}
    Custom server
    \n
    {protocol}://{host}/api/v1/tag/{uuid}
     Request samples 
    • Payload
    • curl
    • Python
    Content type
    application/json
    {
    • \"url\": "http://example.com",
    • \"title\": "string",
    • \"tag\": "string",
    • \"tags\": [
      • "string"
      ],
    • \"paused\": true,
    • \"notification_muted\": true,
    • \"method\": "GET",
    • \"fetch_backend\": "system",
    • \"headers\": {
      • \"property1\": "string",
      • \"property2\": "string"
      },
    • \"body\": "string",
    • \"proxy\": "string",
    • \"ignore_status_codes\": true,
    • \"webdriver_delay\": 0,
    • \"webdriver_js_execute_code\": "string",
    • \"time_between_check\": {
      • \"weeks\": 52000,
      • \"days\": 365000,
      • \"hours\": 8760000,
      • \"minutes\": 525600000,
      • \"seconds\": 31536000000
      },
    • \"time_between_check_use_default\": true,
    • \"notification_urls\": [
      • "string"
      ],
    • \"notification_title\": "string",
    • \"notification_body\": "string",
    • \"notification_format\": "text",
    • \"track_ldjson_price_data\": true,
    • \"browser_steps\": [
      • {
        • \"operation\": "string",
        • \"selector\": "string",
        • \"optional_value\": "string"
        }
      ],
    • \"processor\": "restock_diff",
    • \"include_filters\": [
      • "string"
      ],
    • \"subtractive_selectors\": [
      • "string"
      ],
    • \"ignore_text\": [
      • "string"
      ],
    • \"trigger_text\": [
      • "string"
      ],
    • \"text_should_not_be_present\": [
      • "string"
      ],
    • \"extract_text\": [
      • "string"
      ],
    • \"trim_text_whitespace\": false,
    • \"sort_text_alphabetically\": false,
    • \"remove_duplicate_lines\": false,
    • \"check_unique_lines\": false,
    • \"strip_ignored_lines\": true,
    • \"filter_text_added\": true,
    • \"filter_text_removed\": true,
    • \"filter_text_replaced\": true,
    • \"in_stock_only\": true,
    • \"follow_price_changes\": true,
    • \"price_change_threshold_percent\": 0,
    • \"notification_screenshot\": false,
    • \"filter_failure_notification_send\": true,
    • \"use_page_title_in_list\": true,
    • \"history_snapshot_max_length\": 1,
    • \"time_schedule_limit\": {
      • \"enabled\": false,
      • \"monday\": {
        • \"enabled\": true,
        • \"start_time\": "00:00",
        • \"duration\": {
          • \"hours\": "24",
          • \"minutes\": "00"
          }
        },
      • \"tuesday\": {
        • \"enabled\": true,
        • \"start_time\": "00:00",
        • \"duration\": {
          • \"hours\": "24",
          • \"minutes\": "00"
          }
        },
      • \"wednesday\": {
        • \"enabled\": true,
        • \"start_time\": "00:00",
        • \"duration\": {
          • \"hours\": "24",
          • \"minutes\": "00"
          }
        },
      • \"thursday\": {
        • \"enabled\": true,
        • \"start_time\": "00:00",
        • \"duration\": {
          • \"hours\": "24",
          • \"minutes\": "00"
          }
        },
      • \"friday\": {
        • \"enabled\": true,
        • \"start_time\": "00:00",
        • \"duration\": {
          • \"hours\": "24",
          • \"minutes\": "00"
          }
        },
      • \"saturday\": {
        • \"enabled\": true,
        • \"start_time\": "00:00",
        • \"duration\": {
          • \"hours\": "24",
          • \"minutes\": "00"
          }
        },
      • \"sunday\": {
        • \"enabled\": true,
        • \"start_time\": "00:00",
        • \"duration\": {
          • \"hours\": "24",
          • \"minutes\": "00"
          }
        }
      },
    • \"conditions\": [
      • {
        • \"field\": "string",
        • \"operator\": "string",
        • \"value\": "string"
        }
      ],
    • \"conditions_match_logic\": "ALL",
    • \"overrides_watch\": true
    }
    Delete tag 
    Delete a tag/group and remove it from all web page change monitors (watches)
    \n
    Authorizations:
    ApiKeyAuth
    path Parameters
    uuid
    required
    string <uuid> 
     
    Tag unique ID
    \n
    Responses
    Development server
    \n
    http://localhost:5000/api/v1/tag/{uuid}
    Production server
    \n
    https://yourdomain.com/api/v1/tag/{uuid}
    Custom server
    \n
    {protocol}://{host}/api/v1/tag/{uuid}
     Request samples 
    • curl
    • Python
    curl -X DELETE \"http://localhost:5000/api/v1/tag/550e8400-e29b-41d4-a716-446655440000\" \\\n  -H \"x-api-key: YOUR_API_KEY\"\n
    Notifications
    Configure global notification endpoints that can be used across all your watches. Supports various \nnotification services including email, Discord, Slack, webhooks, and many other popular platforms. \nThese settings serve as defaults that can be overridden at the individual watch or tag level.
    \n
    The notification syntax uses https://github.com/caronc/apprise.
    \n
    Get notification URLs 
    Return the notification URL list from the configuration
    \n
    Authorizations:
    ApiKeyAuth
    Responses
    Development server
    \n
    http://localhost:5000/api/v1/notifications
    Production server
    \n
    https://yourdomain.com/api/v1/notifications
    Custom server
    \n
    {protocol}://{host}/api/v1/notifications
     Request samples 
    • curl
    • Python
    curl -X GET \"http://localhost:5000/api/v1/notifications\" \\\n  -H \"x-api-key: YOUR_API_KEY\"\n
     Response samples 
    • 200
    Content type
    application/json
    {}
    Add notification URLs 
    Add one or more notification URLs to the configuration
    \n
    Authorizations:
    ApiKeyAuth
    Request Body schema: application/json
    required
    notification_urls
    required
    Array of strings <uri> [ items <uri > ]
     
    List of notification URLs
    \n
    Responses
    Development server
    \n
    http://localhost:5000/api/v1/notifications
    Production server
    \n
    https://yourdomain.com/api/v1/notifications
    Custom server
    \n
    {protocol}://{host}/api/v1/notifications
     Request samples 
    • Payload
    • curl
    • Python
    Content type
    application/json
    {
    • \"notification_urls\": [
      • "mailto:admin@example.com",
      • "discord://webhook_id/webhook_token"
      ]
    }
     Response samples 
    • 201
    Content type
    application/json
    {}
    Replace notification URLs 
    Replace all notification URLs with the provided list (can be empty)
    \n
    Authorizations:
    ApiKeyAuth
    Request Body schema: application/json
    required
    notification_urls
    required
    Array of strings <uri> [ items <uri > ]
     
    List of notification URLs
    \n
    Responses
    Development server
    \n
    http://localhost:5000/api/v1/notifications
    Production server
    \n
    https://yourdomain.com/api/v1/notifications
    Custom server
    \n
    {protocol}://{host}/api/v1/notifications
     Request samples 
    • Payload
    • curl
    • Python
    Content type
    application/json
    {}
     Response samples 
    • 200
    Content type
    application/json
    {}
    Delete notification URLs 
    Delete one or more notification URLs from the configuration
    \n
    Authorizations:
    ApiKeyAuth
    Request Body schema: application/json
    required
    notification_urls
    required
    Array of strings <uri> [ items <uri > ]
     
    List of notification URLs
    \n
    Responses
    Development server
    \n
    http://localhost:5000/api/v1/notifications
    Production server
    \n
    https://yourdomain.com/api/v1/notifications
    Custom server
    \n
    {protocol}://{host}/api/v1/notifications
     Request samples 
    • Payload
    • curl
    • Python
    Content type
    application/json
    {}
    Search
    Search and filter your watches by URL patterns, titles, or tags. Useful for quickly finding specific \nmonitors in large collections or identifying watches that match certain criteria.
    \n
    Search watches 
    Search web page change monitors (watches) by URL or title text
    \n
    Authorizations:
    ApiKeyAuth
    query Parameters
    q
    required
    string
     
    Search query to match against watch URLs and titles
    \n
    tag
    string
     
    Tag name to limit results (name not UUID)
    \n
    partial
    string
     
    Allow partial matching of URL query
    \n
    Responses
    Development server
    \n
    http://localhost:5000/api/v1/search
    Production server
    \n
    https://yourdomain.com/api/v1/search
    Custom server
    \n
    {protocol}://{host}/api/v1/search
     Request samples 
    • curl
    • Python
    curl -X GET \"http://localhost:5000/api/v1/search?q=example.com\" \\\n  -H \"x-api-key: YOUR_API_KEY\"\n
     Response samples 
    • 200
    Content type
    application/json
    {
    • \"watches\": {
      • \"095be615-a8ad-4c33-8e9c-c7612fbf6c9f\": {
        • \"uuid\": "095be615-a8ad-4c33-8e9c-c7612fbf6c9f",
        • \"url\": "http://example.com",
        • \"title\": "Example Website Monitor",
        • \"tags\": [
          • "550e8400-e29b-41d4-a716-446655440000"
          ],
        • \"paused\": false,
        • \"notification_muted\": false
        }
      }
    }
    Import
    Bulk import multiple URLs for monitoring. Accepts plain text lists of URLs and can automatically \napply tags, proxy settings, and other configurations to all imported watches simultaneously.
    \n
    Import watch URLs with configuration 
    Import a list of URLs to monitor with optional watch configuration. Accepts line-separated URLs in request body.
    \n
    Configuration via Query Parameters:
    \n
    You can pass ANY watch configuration field as query parameters to apply settings to all imported watches.\nAll parameters from the Watch schema are supported (processor, fetch_backend, notification_urls, etc.).
    \n
    Special Parameters:
    \n
      \n
    • tag / tag_uuids - Assign tags to imported watches
      • \n
    • proxy - Use specific proxy for imported watches
      • \n
    • dedupe - Skip duplicate URLs (default: true)
      • \n
    \n
    Type Conversion:
    \n
      \n
    • Booleans: true, false, 1, 0, yes, no
      • \n
    • Arrays: Comma-separated or JSON format ([item1,item2])
      • \n
    • Objects: JSON format ({"key":"value"})
      • \n
    • Numbers: Parsed as int or float
      • \n
    \n
    Authorizations:
    ApiKeyAuth
    query Parameters
    tag_uuids
    string
     
     Example:  tag_uuids=550e8400-e29b-41d4-a716-446655440000
    Tag UUID(s) to apply to imported watches (comma-separated for multiple)
    \n
    tag
    string
     
     Example:  tag=production
    Tag name to apply to imported watches
    \n
    proxy
    string
     
     Example:  proxy=proxy1
    Proxy key to use for imported watches
    \n
    dedupe
    boolean
     Default:  true
     
    Skip duplicate URLs (default true)
    \n
    Request Body schema: text/plain
    required
    string
     
    Responses
    Development server
    \n
    http://localhost:5000/api/v1/import
    Production server
    \n
    https://yourdomain.com/api/v1/import
    Custom server
    \n
    {protocol}://{host}/api/v1/import
     Request samples 
    • Payload
    • curl
    • Python
    Content type
    text/plain
    https://example.com\nhttps://example.org\nhttps://example.net\n
     Response samples 
    • 200
    Content type
    application/json
    [
    • "497f6eca-6276-4993-bfeb-53cbbbba6f08"
    ]
    System Information
    Retrieve system status and statistics about your changedetection.io instance, including total watch\ncounts, uptime information, and version details.
    \n
    Get system information 
    Return information about the current system state
    \n
    Authorizations:
    ApiKeyAuth
    Responses
    Development server
    \n
    http://localhost:5000/api/v1/systeminfo
    Production server
    \n
    https://yourdomain.com/api/v1/systeminfo
    Custom server
    \n
    {protocol}://{host}/api/v1/systeminfo
     Request samples 
    • curl
    • Python
    curl -X GET \"http://localhost:5000/api/v1/systeminfo\" \\\n  -H \"x-api-key: YOUR_API_KEY\"\n
     Response samples 
    • 200
    Content type
    application/json
    {
    • \"watch_count\": 42,
    • \"tag_count\": 5,
    • \"uptime\": "2 days, 3:45:12",
    • \"version\": "0.50.10"
    }
    Plugin API Extensions
    How Processor Plugins Extend the API
    changedetection.io uses a processor plugin system to handle different types of change detection.\nEach processor lives in changedetectionio/processors/<name>/ and may include an api.yaml file\nthat extends the core Watch schema with processor-specific configuration fields.
    \n
    How it works
    \n
    At startup, changedetection.io scans all installed processors for an api.yaml file. Any schemas\nand code samples defined there are deep-merged into the live API specification, making the\nprocessor's configuration fields valid on all watch create and update requests.
    \n
    The live, fully-merged spec is always available at /api/v1/full-spec — use that URL with\nSwagger UI or Redoc to see the complete schema for your specific installation.
    \n
    \n
    Writing a processor api.yaml
    \n
    Place an api.yaml in the processor plugin's own directory, alongside its __init__.py\n(e.g. changedetectionio/processors/my_processor/api.yaml). The schema name must follow the\nconvention processor_config_<processor_name> (e.g. processor_config_restock_diff). That same\nkey is used as the JSON field name when creating or updating a watch.
    \n
    A minimal api.yaml for a hypothetical my_processor:
    \n
    components:\n  schemas:\n    processor_config_my_processor:\n      type: object\n      description: Configuration for my_processor\n      properties:\n        some_option:\n          type: boolean\n          default: true\n          description: Enable some behaviour\n\npaths:\n  /watch:\n    post:\n      x-code-samples:\n        - lang: curl\n          label: my_processor example\n          source: |\n            curl -X POST \"http://localhost:5000/api/v1/watch\" \\\n              -H \"x-api-key: YOUR_API_KEY\" \\\n              -H \"Content-Type: application/json\" \\\n              -d '{\n                \"url\": \"https://example.com\",\n                \"processor\": \"my_processor\",\n                \"processor_config_my_processor\": { \"some_option\": true }\n              }'\n
    \n
    The paths section in api.yaml is used only for injecting additional x-code-samples into\nexisting endpoints — you cannot define new routes via plugin.
    \n
    \n
    Built-in plugin: restock_diff
    \n
    The restock_diff processor is always shipped with changedetection.io. It monitors product\navailability and price changes using structured data (JSON-LD / schema.org microdata) and\ntext heuristics. It is activated by setting "processor": "restock_diff" on a watch.
    \n
    It adds the processor_config_restock_diff block to the Watch schema with these fields:
    \n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
    FieldTypeDefaultDescription
    in_stock_processingstringin_stock_onlyin_stock_only — only alert Out-of-Stock→In-Stock · all_changes — alert any availability change · off — disable stock tracking
    follow_price_changesbooleantrueMonitor and alert on price changes
    price_change_minnumber|null—Alert when price drops below this value
    price_change_maxnumber|null—Alert when price rises above this value
    price_change_threshold_percentnumber|null—Minimum % change since the original price to trigger an alert
    \n
    CREATE — Add a restock/price monitor
    \n
    curl -X POST \"http://localhost:5000/api/v1/watch\" \\\n  -H \"x-api-key: YOUR_API_KEY\" \\\n  -H \"Content-Type: application/json\" \\\n  -d '{\n    \"url\": \"https://example.com/product/widget\",\n    \"processor\": \"restock_diff\",\n    \"processor_config_restock_diff\": {\n      \"in_stock_processing\": \"in_stock_only\",\n      \"follow_price_changes\": true,\n      \"price_change_threshold_percent\": 5\n    }\n  }'\n
    \n
    READ — Retrieve the monitor
    \n
    The response JSON includes processor_config_restock_diff alongside all standard watch fields:
    \n
    curl -X GET \"http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091\" \\\n  -H \"x-api-key: YOUR_API_KEY\"\n
    \n
    {\n  \"uuid\": \"cc0cfffa-f449-477b-83ea-0caafd1dc091\",\n  \"url\": \"https://example.com/product/widget\",\n  \"processor\": \"restock_diff\",\n  \"processor_config_restock_diff\": {\n    \"in_stock_processing\": \"in_stock_only\",\n    \"follow_price_changes\": true,\n    \"price_change_threshold_percent\": 5,\n    \"price_change_min\": null,\n    \"price_change_max\": null\n  }\n}\n
    \n
    UPDATE — Change thresholds without recreating the monitor
    \n
    Only fields included in the request body are updated; omitted fields are left unchanged.
    \n
    curl -X PUT \"http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091\" \\\n  -H \"x-api-key: YOUR_API_KEY\" \\\n  -H \"Content-Type: application/json\" \\\n  -d '{\n    \"processor_config_restock_diff\": {\n      \"in_stock_processing\": \"all_changes\",\n      \"follow_price_changes\": true,\n      \"price_change_min\": 10.00,\n      \"price_change_max\": 500.00\n    }\n  }'\n
    \n
    DELETE — Remove the monitor
    \n
    curl -X DELETE \"http://localhost:5000/api/v1/watch/cc0cfffa-f449-477b-83ea-0caafd1dc091\" \\\n  -H \"x-api-key: YOUR_API_KEY\"\n
    \n
    \n
    For the complete schema-validated documentation including all processor fields, fetch the live spec\nand load it into Swagger UI or Redoc:
    \n
    GET /api/v1/full-spec\n
    \n
    Get full live API spec 
    Return the fully merged OpenAPI specification for this instance.
    \n
    Unlike the static api-spec.yaml shipped with the application, this endpoint returns the\nspec dynamically merged with any api.yaml schemas provided by installed processor plugins.
    \n
    Use this URL with Swagger UI or Redoc to get schema-accurate documentation for your\nspecific install — it includes every processor_config_<name> schema block contributed by\ninstalled processors (e.g. processor_config_restock_diff from the built-in restock plugin).
    \n
    This endpoint requires no authentication and returns YAML.
    \n
    To load it directly in Swagger UI, paste the URL into the "Explore" box:
    \n
    http://localhost:5000/api/v1/full-spec\n
    \n
    Responses
    Development server
    \n
    http://localhost:5000/api/v1/full-spec
    Production server
    \n
    https://yourdomain.com/api/v1/full-spec
    Custom server
    \n
    {protocol}://{host}/api/v1/full-spec
     Request samples 
    • curl
    • Python
    # Fetch the live merged spec (no API key needed)\ncurl -X GET \"http://localhost:5000/api/v1/full-spec\"\n
    \n      \n\n\n\n"
  },
  {
    "path": "docs/package.json",
    "content": "{\n  \"name\": \"changedetection-api-docs\",\n  \"version\": \"1.0.0\",\n  \"description\": \"API documentation generation for changedetection.io\",\n  \"private\": true,\n  \"scripts\": {\n    \"build-docs\": \"redocly build-docs api-spec.yaml --output api_v1/index.html\"\n  },\n  \"devDependencies\": {\n    \"@redocly/cli\": \"^1.34.5\"\n  }\n}"
  },
  {
    "path": "requirements.txt",
    "content": "feedgen~=1.0\nfeedparser~=6.0  # For parsing RSS/Atom feeds\nflask-compress\nflask-login>=0.6.3\nflask-paginate\nflask-socketio>=5.6.1,<6 # Re #3910\nflask>=3.1,<4\nflask_cors # For the Chrome extension to operate\nflask_restful\nflask_wtf~=1.2\ninscriptis~=2.2\npython-engineio>=4.9.0,<5\npython-socketio>=5.11.0,<6\npytz\ntimeago~=1.0\nvalidators~=0.35\nwerkzeug==3.1.6\n\n\n# Set these versions together to avoid a RequestsDependencyWarning\n# >= 2.26 also adds Brotli support if brotli is installed\nbrotli~=1.2\nrequests[socks]\nrequests-file\n\n# urllib3==1.26.19  # Unpinned - let requests decide compatible version\n# If specific version needed for security, use urllib3>=1.26.19,<3.0\nchardet>2.3.0\n\nwtforms~=3.2\njsonpath-ng~=1.8.0\n\n# Fast JSON serialization for better performance\norjson~=3.11\n\n# dnspython - Used by paho-mqtt for MQTT broker resolution  \n# Version pin removed since eventlet (which required the specific 2.6.1 pin) has been eliminated\n# paho-mqtt will install compatible dnspython version automatically\n\n# jq not available on Windows so must be installed manually\n\n# Notification library\napprise==1.9.8\n\ndiff_match_patch\n\n# Lightweight URL linkifier for notifications\nlinkify-it-py\n\n# - Needed for apprise/spush, and maybe others? hopefully doesnt trigger a rust compile.\n# - Requires extra wheel for rPi, adds build time for arm/v8 which is not in piwheels\n# Pinned to 44.x for ARM compatibility and sslyze compatibility (sslyze requires <45) and (45.x may not have pre-built ARM wheels)\n# Also pinned because dependabot wants specific versions\ncryptography==44.0.0\n\n# apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315\n# use any version other than 2.0.x due to https://github.com/eclipse/paho.mqtt.python/issues/814\npaho-mqtt!=2.0.*\n\n# Used for CSS filtering, JSON extraction from HTML\nbeautifulsoup4>=4.0.0,<=4.14.3\n\n# XPath filtering, lxml is required by bs4 anyway, but put it here to be safe.\n# #2328 - 5.2.0 and 5.2.1 had extra CPU flag CFLAGS set which was not compatible on older hardware\n#         It could be advantageous to run its own pypi package here with those performance flags set\n#         https://bugs.launchpad.net/lxml/+bug/2059910/comments/16\nlxml >=4.8.0,!=5.2.0,!=5.2.1,<7\n\n# XPath 2.0-3.1 support - 4.2.0 had issues, 4.1.5 stable\n# Consider updating to latest stable version periodically\nelementpath==5.1.1\n\n# For fast image comparison in screenshot change detection\n# opencv-python-headless is OPTIONAL (excluded from requirements.txt)\n# - Installed conditionally via Dockerfile (skipped on arm/v7 and arm/v8 due to long build times)\n# - Pixelmatch is used as fallback when OpenCV is unavailable\n# - To install manually: pip install opencv-python-headless>=4.8.0.76\n\nselenium~=4.31.0\n\n# Templating, so far just in the URLs but in the future can be for the notifications also\njinja2~=3.1\narrow\nopenpyxl\n# https://peps.python.org/pep-0508/#environment-markers\n# https://github.com/dgtlmoon/changedetection.io/pull/1009\njq~=1.3; python_version >= \"3.8\" and sys_platform == \"darwin\"\njq~=1.3; python_version >= \"3.8\" and sys_platform == \"linux\"\n\n# playwright is installed at Dockerfile build time because it's not available on all platforms\n\npyppeteer-ng==2.0.0rc13\npyppeteerstealth>=0.0.4\n\n# Include pytest, so if theres a support issue we can ask them to run these tests on their setup\npytest ~=9.0\npytest-flask ~=1.3\npytest-mock ~=3.15\n\n# OpenAPI validation support\nopenapi-core[flask] ~= 0.22\n\nloguru\n\n# For scraping all possible metadata relating to products so we can do better restock detection\nextruct\n\n# For cleaning up unknown currency formats\nbabel\n# For internationalization (i18n) support\nFlask-Babel>=4.0.0\n\nlevenshtein\n\n# Needed for > 3.10, https://github.com/microsoft/playwright-python/issues/2096\ngreenlet >= 3.0.3\n\n# Optional: Used for high-concurrency SocketIO mode (via SOCKETIO_MODE=gevent)\n# Note: gevent has cross-platform limitations (Windows 1024 socket limit, macOS ARM build issues)\n# Default SOCKETIO_MODE=threading is recommended for better compatibility\ngevent\n\nreferencing  # Don't pin — jsonschema-path (required by openapi-core>=0.18) caps referencing<0.37.0, so pinning 0.37.0 forces openapi-core back to 0.17.2. Revisit once jsonschema-path>=0.3.5 relaxes the cap.\n\n# For conditions\npanzi-json-logic\n# For conditions - extracted number from a body of text\nprice-parser\n\n# flask_socket_io - incorrect package name, already have flask-socketio above\n\n# Lightweight MIME type detection (saves ~14MB memory vs python-magic/libmagic)\n# Used for detecting correct favicon type and content-type detection\npuremagic\n\n# Scheduler - Windows seemed to miss a lot of default timezone info (even \"UTC\" !)\ntzdata\n\n#typing_extensions ==4.8.0\n\npluggy ~= 1.6\n\n# Needed for testing, cross-platform for process and system monitoring\npsutil==7.2.2\n\nruff >= 0.11.2\npre_commit >= 4.2.0\n\n# For events between checking and socketio updates\nblinker\npytest-xdist\n\n\n"
  },
  {
    "path": "runtime.txt",
    "content": "python-3.11.5"
  },
  {
    "path": "setup.cfg",
    "content": "# Translation configuration for changedetection.io\n# See changedetectionio/translations/README.md for full documentation on updating translations\n\n[extract_messages]\n# Extract translatable strings from source code\nmapping_file = babel.cfg\noutput_file = changedetectionio/translations/messages.pot\ninput_paths = changedetectionio\nkeywords = _ _l gettext\n# Options to reduce unnecessary changes in .pot files\nsort_by_file = true\nwidth = 120\nadd_location = file\n\n[update_catalog]\n# Update existing .po files with new strings from .pot\n# Note: Omitting 'locale' makes Babel auto-discover all catalogs in output_dir\ninput_file = changedetectionio/translations/messages.pot\noutput_dir = changedetectionio/translations\ndomain = messages\n# Options for consistent formatting\nwidth = 120\nno_fuzzy_matching = true\n\n[compile_catalog]\n# Compile .po files to .mo binary format\ndirectory = changedetectionio/translations\ndomain = messages\n"
  },
  {
    "path": "setup.py",
    "content": "#!/usr/bin/env python\nimport codecs\nimport os.path\nimport re\nimport sys\n\nfrom setuptools import setup, find_packages\nfrom setuptools.command.build_py import build_py\nimport shutil\n\nhere = os.path.abspath(os.path.dirname(__file__))\n\n\ndef read(*parts):\n    return codecs.open(os.path.join(here, *parts), 'r').read()\n\n\ndef find_version(*file_paths):\n    version_file = read(*file_paths)\n    version_match = re.search(r\"^__version__ = ['\\\"]([^'\\\"]*)['\\\"]\",\n                              version_file, re.M)\n    if version_match:\n        return version_match.group(1)\n    raise RuntimeError(\"Unable to find version string.\")\n\n\nclass BuildPyCommand(build_py):\n    \"\"\"Custom build command to copy api-spec.yaml to the package.\"\"\"\n    def run(self):\n        build_py.run(self)\n        # Ensure the docs directory exists in the build output\n        docs_dir = os.path.join(self.build_lib, 'changedetectionio', 'docs')\n        os.makedirs(docs_dir, exist_ok=True)\n        # Copy api-spec.yaml to the package\n        shutil.copy(\n            os.path.join(here, 'docs', 'api-spec.yaml'),\n            os.path.join(docs_dir, 'api-spec.yaml')\n        )\n\n\ninstall_requires = open('requirements.txt').readlines()\n\nsetup(\n    name='changedetection.io',\n    version=find_version(\"changedetectionio\", \"__init__.py\"),\n    description='Website change detection and monitoring service, detect changes to web pages and send alerts/notifications.',\n    long_description=open('README-pip.md').read(),\n    long_description_content_type='text/markdown',\n    keywords='website change monitor for changes notification change detection '\n             'alerts tracking website tracker change alert website and monitoring',\n    entry_points={\"console_scripts\": [\"changedetection.io=changedetectionio:main\"]},\n    zip_safe=True,\n    scripts=[\"changedetection.py\"],\n    author='dgtlmoon',\n    url='https://changedetection.io',\n    packages=find_packages(include=['changedetectionio', 'changedetectionio.*']),\n    include_package_data=True,\n    install_requires=install_requires,\n    cmdclass={'build_py': BuildPyCommand},\n    license=\"Apache License 2.0\",\n    python_requires=\">= 3.10\",\n    classifiers=['Intended Audience :: Customer Service',\n                 'Intended Audience :: Developers',\n                 'Intended Audience :: Education',\n                 'Intended Audience :: End Users/Desktop',\n                 'Intended Audience :: Financial and Insurance Industry',\n                 'Intended Audience :: Healthcare Industry',\n                 'Intended Audience :: Information Technology',\n                 'Intended Audience :: Legal Industry',\n                 'Intended Audience :: Manufacturing',\n                 'Intended Audience :: Other Audience',\n                 'Intended Audience :: Religion',\n                 'Intended Audience :: Science/Research',\n                 'Intended Audience :: System Administrators',\n                 'Intended Audience :: Telecommunications Industry',\n                 'Topic :: Education',\n                 'Topic :: Internet',\n                 'Topic :: Internet :: WWW/HTTP :: Indexing/Search',\n                 'Topic :: Internet :: WWW/HTTP :: Site Management',\n                 'Topic :: Internet :: WWW/HTTP :: Site Management :: Link Checking',\n                 'Topic :: Internet :: WWW/HTTP :: Browsers',\n                 'Topic :: Internet :: WWW/HTTP :: Dynamic Content',\n                 'Topic :: Office/Business',\n                 'Topic :: Other/Nonlisted Topic',\n                 'Topic :: Scientific/Engineering :: Information Analysis',\n                 'Topic :: Text Processing :: Markup :: HTML',\n                 'Topic :: Utilities'\n                 ],\n)\n"
  }
]