[ { "path": ".coveragerc", "content": "[run]\nomit = datasette/_version.py, datasette/utils/shutil_backport.py\n" }, { "path": ".dockerignore", "content": ".DS_Store\n.cache\n.eggs\n.gitignore\n.ipynb_checkpoints\nbuild\n*.spec\n*.egg-info\ndist\nscratchpad\nvenv\n*.db\n*.sqlite\n" }, { "path": ".git-blame-ignore-revs", "content": "# Applying Black\n35d6ee2790e41e96f243c1ff58be0c9c0519a8ce\n368638555160fb9ac78f462d0f79b1394163fa30\n2b344f6a34d2adaa305996a1a580ece06397f6e4\n" }, { "path": ".gitattributes", "content": "datasette/static/codemirror-* linguist-vendored\n" }, { "path": ".github/FUNDING.yml", "content": "github: [simonw]\n" }, { "path": ".github/dependabot.yml", "content": "version: 2\nupdates:\n- package-ecosystem: pip\n directory: \"/\"\n schedule:\n interval: daily\n time: \"13:00\"\n groups:\n python-packages:\n patterns:\n - \"*\"\n" }, { "path": ".github/workflows/deploy-branch-preview.yml", "content": "name: Deploy a Datasette branch preview to Vercel\n\non:\n workflow_dispatch:\n inputs:\n branch:\n description: \"Branch to deploy\"\n required: true\n type: string\n\njobs:\n deploy-branch-preview:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v3\n - name: Set up Python 3.11\n uses: actions/setup-python@v6\n with:\n python-version: \"3.11\"\n - name: Install dependencies\n run: |\n pip install datasette-publish-vercel\n - name: Deploy the preview\n env:\n VERCEL_TOKEN: ${{ secrets.BRANCH_PREVIEW_VERCEL_TOKEN }}\n run: |\n export BRANCH=\"${{ github.event.inputs.branch }}\"\n wget https://latest.datasette.io/fixtures.db\n datasette publish vercel fixtures.db \\\n --branch $BRANCH \\\n --project \"datasette-preview-$BRANCH\" \\\n --token $VERCEL_TOKEN \\\n --scope datasette \\\n --about \"Preview of $BRANCH\" \\\n --about_url \"https://github.com/simonw/datasette/tree/$BRANCH\"\n" }, { "path": ".github/workflows/deploy-latest.yml", "content": "name: Deploy latest.datasette.io\n\non:\n workflow_dispatch:\n push:\n branches:\n - main\n # - 1.0-dev\n\npermissions:\n contents: read\n\njobs:\n deploy:\n runs-on: ubuntu-latest\n steps:\n - name: Check out datasette\n uses: actions/checkout@v5\n - name: Set up Python\n uses: actions/setup-python@v6\n with:\n python-version: \"3.13\"\n cache: pip\n - name: Install Python dependencies\n run: |\n python -m pip install --upgrade pip\n python -m pip install . --group dev\n python -m pip install sphinx-to-sqlite==0.1a1\n - name: Run tests\n if: ${{ github.ref == 'refs/heads/main' }}\n run: |\n pytest -n auto -m \"not serial\"\n pytest -m \"serial\"\n - name: Build fixtures.db and other files needed to deploy the demo\n run: |-\n python tests/fixtures.py \\\n fixtures.db \\\n fixtures-config.json \\\n fixtures-metadata.json \\\n plugins \\\n --extra-db-filename extra_database.db\n - name: Build docs.db\n if: ${{ github.ref == 'refs/heads/main' }}\n run: |-\n cd docs\n DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build\n sphinx-to-sqlite ../docs.db _build\n cd ..\n - name: Set up the alternate-route demo\n run: |\n echo '\n from datasette import hookimpl\n\n @hookimpl\n def startup(datasette):\n db = datasette.get_database(\"fixtures2\")\n db.route = \"alternative-route\"\n ' > plugins/alternative_route.py\n cp fixtures.db fixtures2.db\n - name: And the counters writable canned query demo\n run: |\n cat > plugins/counters.py < metadata.json\n # cat metadata.json\n - id: auth\n name: Authenticate to Google Cloud\n uses: google-github-actions/auth@v3\n with:\n credentials_json: ${{ secrets.GCP_SA_KEY }}\n - name: Set up Cloud SDK\n uses: google-github-actions/setup-gcloud@v3\n - name: Deploy to Cloud Run\n env:\n LATEST_DATASETTE_SECRET: ${{ secrets.LATEST_DATASETTE_SECRET }}\n run: |-\n gcloud config set run/region us-central1\n gcloud config set project datasette-222320\n export SUFFIX=\"-${GITHUB_REF#refs/heads/}\"\n export SUFFIX=${SUFFIX#-main}\n # Replace 1.0 with one-dot-zero in SUFFIX\n export SUFFIX=${SUFFIX//1.0/one-dot-zero}\n datasette publish cloudrun fixtures.db fixtures2.db extra_database.db \\\n -m fixtures-metadata.json \\\n --plugins-dir=plugins \\\n --branch=$GITHUB_SHA \\\n --version-note=$GITHUB_SHA \\\n --extra-options=\"--setting template_debug 1 --setting trace_debug 1 --crossdb\" \\\n --install 'datasette-ephemeral-tables>=0.2.2' \\\n --service \"datasette-latest$SUFFIX\" \\\n --secret $LATEST_DATASETTE_SECRET\n - name: Deploy to docs as well (only for main)\n if: ${{ github.ref == 'refs/heads/main' }}\n run: |-\n # Deploy docs.db to a different service\n datasette publish cloudrun docs.db \\\n --branch=$GITHUB_SHA \\\n --version-note=$GITHUB_SHA \\\n --extra-options=\"--setting template_debug 1\" \\\n --service=datasette-docs-latest\n" }, { "path": ".github/workflows/documentation-links.yml", "content": "name: Read the Docs Pull Request Preview\non:\n pull_request_target:\n types:\n - opened\n\npermissions:\n pull-requests: write\n\njobs:\n documentation-links:\n runs-on: ubuntu-latest\n steps:\n - uses: readthedocs/actions/preview@v1\n with:\n project-slug: \"datasette\"\n" }, { "path": ".github/workflows/prettier.yml", "content": "name: Check JavaScript for conformance with Prettier\n\non: [push]\n\npermissions:\n contents: read\n\njobs:\n prettier:\n runs-on: ubuntu-latest\n steps:\n - name: Check out repo\n uses: actions/checkout@v4\n - uses: actions/cache@v4\n name: Configure npm caching\n with:\n path: ~/.npm\n key: ${{ runner.OS }}-npm-${{ hashFiles('**/package-lock.json') }}\n restore-keys: |\n ${{ runner.OS }}-npm-\n - name: Install dependencies\n run: npm ci\n - name: Run prettier\n run: |-\n npm run prettier -- --check\n" }, { "path": ".github/workflows/publish.yml", "content": "name: Publish Python Package\n\non:\n release:\n types: [created]\n\npermissions:\n contents: read\n\njobs:\n test:\n runs-on: ubuntu-latest\n strategy:\n matrix:\n python-version: [\"3.10\", \"3.11\", \"3.12\", \"3.13\", \"3.14\"]\n steps:\n - uses: actions/checkout@v4\n - name: Set up Python ${{ matrix.python-version }}\n uses: actions/setup-python@v6\n with:\n python-version: ${{ matrix.python-version }}\n cache: pip\n cache-dependency-path: pyproject.toml\n - name: Install dependencies\n run: |\n pip install . --group dev\n - name: Run tests\n run: |\n pytest\n\n deploy:\n runs-on: ubuntu-latest\n needs: [test]\n environment: release\n permissions:\n id-token: write\n steps:\n - uses: actions/checkout@v4\n - name: Set up Python\n uses: actions/setup-python@v6\n with:\n python-version: '3.13'\n cache: pip\n cache-dependency-path: pyproject.toml\n - name: Install dependencies\n run: |\n pip install setuptools wheel build\n - name: Build\n run: |\n python -m build\n - name: Publish\n uses: pypa/gh-action-pypi-publish@release/v1\n\n deploy_static_docs:\n runs-on: ubuntu-latest\n needs: [deploy]\n if: \"!github.event.release.prerelease\"\n steps:\n - uses: actions/checkout@v4\n - name: Set up Python\n uses: actions/setup-python@v6\n with:\n python-version: '3.10'\n cache: pip\n cache-dependency-path: pyproject.toml\n - name: Install dependencies\n run: |\n python -m pip install . --group dev\n python -m pip install sphinx-to-sqlite==0.1a1\n - name: Build docs.db\n run: |-\n cd docs\n DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build\n sphinx-to-sqlite ../docs.db _build\n cd ..\n - id: auth\n name: Authenticate to Google Cloud\n uses: google-github-actions/auth@v2\n with:\n credentials_json: ${{ secrets.GCP_SA_KEY }}\n - name: Set up Cloud SDK\n uses: google-github-actions/setup-gcloud@v3\n - name: Deploy stable-docs.datasette.io to Cloud Run\n run: |-\n gcloud config set run/region us-central1\n gcloud config set project datasette-222320\n datasette publish cloudrun docs.db \\\n --service=datasette-docs-stable\n\n deploy_docker:\n runs-on: ubuntu-latest\n needs: [deploy]\n if: \"!github.event.release.prerelease\"\n steps:\n - uses: actions/checkout@v4\n - name: Build and push to Docker Hub\n env:\n DOCKER_USER: ${{ secrets.DOCKER_USER }}\n DOCKER_PASS: ${{ secrets.DOCKER_PASS }}\n run: |-\n sleep 60 # Give PyPI time to make the new release available\n docker login -u $DOCKER_USER -p $DOCKER_PASS\n export REPO=datasetteproject/datasette\n docker build -f Dockerfile \\\n -t $REPO:${GITHUB_REF#refs/tags/} \\\n --build-arg VERSION=${GITHUB_REF#refs/tags/} .\n docker tag $REPO:${GITHUB_REF#refs/tags/} $REPO:latest\n docker push $REPO:${GITHUB_REF#refs/tags/}\n docker push $REPO:latest\n" }, { "path": ".github/workflows/push_docker_tag.yml", "content": "name: Push specific Docker tag\n\non:\n workflow_dispatch:\n inputs:\n version_tag:\n description: Tag to build and push\n\npermissions:\n contents: read\n\njobs:\n deploy_docker:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v2\n - name: Build and push to Docker Hub\n env:\n DOCKER_USER: ${{ secrets.DOCKER_USER }}\n DOCKER_PASS: ${{ secrets.DOCKER_PASS }}\n VERSION_TAG: ${{ github.event.inputs.version_tag }}\n run: |-\n docker login -u $DOCKER_USER -p $DOCKER_PASS\n export REPO=datasetteproject/datasette\n docker build -f Dockerfile \\\n -t $REPO:${VERSION_TAG} \\\n --build-arg VERSION=${VERSION_TAG} .\n docker push $REPO:${VERSION_TAG}\n" }, { "path": ".github/workflows/spellcheck.yml", "content": "name: Check spelling in documentation\n\non: [push, pull_request]\n\npermissions:\n contents: read\n\njobs:\n spellcheck:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v4\n - name: Set up Python\n uses: actions/setup-python@v6\n with:\n python-version: '3.11'\n cache: 'pip'\n cache-dependency-path: '**/pyproject.toml'\n - name: Install dependencies\n run: |\n pip install . --group dev\n - name: Check spelling\n run: |\n codespell README.md --ignore-words docs/codespell-ignore-words.txt\n codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt\n codespell datasette -S datasette/static --ignore-words docs/codespell-ignore-words.txt\n codespell tests --ignore-words docs/codespell-ignore-words.txt\n" }, { "path": ".github/workflows/stable-docs.yml", "content": "name: Update Stable Docs\n\non:\n release:\n types: [published]\n push:\n branches:\n - main\n\npermissions:\n contents: write\n\njobs:\n update_stable_docs:\n runs-on: ubuntu-latest\n steps:\n - name: Checkout repository\n uses: actions/checkout@v5\n with:\n fetch-depth: 0 # We need all commits to find docs/ changes\n - name: Set up Git user\n run: |\n git config user.name \"Automated\"\n git config user.email \"actions@users.noreply.github.com\"\n - name: Create stable branch if it does not yet exist\n run: |\n if ! git ls-remote --heads origin stable | grep -qE '\\bstable\\b'; then\n # Make sure we have all tags locally\n git fetch --tags --quiet\n\n # Latest tag that is just numbers and dots (optionally prefixed with 'v')\n # e.g., 0.65.2 or v0.65.2 — excludes 1.0a20, 1.0-rc1, etc.\n LATEST_RELEASE=$(\n git tag -l --sort=-v:refname \\\n | grep -E '^v?[0-9]+(\\.[0-9]+){1,3}$' \\\n | head -n1\n )\n\n git checkout -b stable\n\n # If there are any stable releases, copy docs/ from the most recent\n if [ -n \"$LATEST_RELEASE\" ]; then\n rm -rf docs/\n git checkout \"$LATEST_RELEASE\" -- docs/ || true\n fi\n\n git commit -m \"Populate docs/ from $LATEST_RELEASE\" || echo \"No changes\"\n git push -u origin stable\n fi\n - name: Handle Release\n if: github.event_name == 'release' && !github.event.release.prerelease\n run: |\n git fetch --all\n git checkout stable\n git reset --hard ${GITHUB_REF#refs/tags/}\n git push origin stable --force\n - name: Handle Commit to Main\n if: contains(github.event.head_commit.message, '!stable-docs')\n run: |\n git fetch origin\n git checkout -b stable origin/stable\n # Get the list of modified files in docs/ from the current commit\n FILES=$(git diff-tree --no-commit-id --name-only -r ${{ github.sha }} -- docs/)\n # Check if the list of files is non-empty\n if [[ -n \"$FILES\" ]]; then\n # Checkout those files to the stable branch to over-write with their contents\n for FILE in $FILES; do\n git checkout ${{ github.sha }} -- $FILE\n done\n git add docs/\n git commit -m \"Doc changes from ${{ github.sha }}\"\n git push origin stable\n else\n echo \"No changes to docs/ in this commit.\"\n exit 0\n fi\n" }, { "path": ".github/workflows/test-coverage.yml", "content": "name: Calculate test coverage\n\non:\n push:\n branches:\n - main\n pull_request:\n branches:\n - main\npermissions:\n contents: read\n\njobs:\n test:\n runs-on: ubuntu-latest\n steps:\n - name: Check out datasette\n uses: actions/checkout@v4\n - name: Set up Python\n uses: actions/setup-python@v6\n with:\n python-version: '3.12'\n cache: 'pip'\n cache-dependency-path: '**/pyproject.toml'\n - name: Install Python dependencies\n run: |\n python -m pip install --upgrade pip\n python -m pip install . --group dev\n python -m pip install pytest-cov\n - name: Run tests\n run: |-\n ls -lah\n cat .coveragerc\n pytest -m \"not serial\" --cov=datasette --cov-config=.coveragerc --cov-report xml:coverage.xml --cov-report term -x\n ls -lah\n - name: Upload coverage report\n uses: codecov/codecov-action@v1\n with:\n token: ${{ secrets.CODECOV_TOKEN }}\n file: coverage.xml\n" }, { "path": ".github/workflows/test-pyodide.yml", "content": "name: Test in Pyodide with shot-scraper\n\non:\n push:\n pull_request:\n workflow_dispatch:\n\npermissions:\n contents: read\n\njobs:\n test:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v4\n - name: Set up Python 3.10\n uses: actions/setup-python@v6\n with:\n python-version: \"3.10\"\n cache: 'pip'\n cache-dependency-path: '**/pyproject.toml'\n - name: Cache Playwright browsers\n uses: actions/cache@v4\n with:\n path: ~/.cache/ms-playwright/\n key: ${{ runner.os }}-browsers\n - name: Install Playwright dependencies\n run: |\n pip install shot-scraper build\n shot-scraper install\n - name: Run test\n run: |\n ./test-in-pyodide-with-shot-scraper.sh\n" }, { "path": ".github/workflows/test-sqlite-support.yml", "content": "name: Test SQLite versions\n\non: [push, pull_request]\n\npermissions:\n contents: read\n\njobs:\n test:\n runs-on: ${{ matrix.platform }}\n continue-on-error: true\n strategy:\n matrix:\n platform: [ubuntu-latest]\n python-version: [\"3.10\", \"3.11\", \"3.12\", \"3.13\", \"3.14\"]\n sqlite-version: [\n #\"3\", # latest version\n \"3.46\",\n #\"3.45\",\n #\"3.27\",\n #\"3.26\",\n \"3.25\",\n #\"3.25.3\", # 2018-09-25, window functions breaks test_upsert for some reason on 3.10, skip for now\n #\"3.24\", # 2018-06-04, added UPSERT support\n #\"3.23.1\" # 2018-04-10, before UPSERT\n ]\n steps:\n - uses: actions/checkout@v4\n - name: Set up Python ${{ matrix.python-version }}\n uses: actions/setup-python@v6\n with:\n python-version: ${{ matrix.python-version }}\n allow-prereleases: true\n cache: pip\n cache-dependency-path: pyproject.toml\n - name: Set up SQLite ${{ matrix.sqlite-version }}\n uses: asg017/sqlite-versions@71ea0de37ae739c33e447af91ba71dda8fcf22e6\n with:\n version: ${{ matrix.sqlite-version }}\n cflags: \"-DSQLITE_ENABLE_DESERIALIZE -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_JSON1\"\n - run: python3 -c \"import sqlite3; print(sqlite3.sqlite_version)\"\n - run: echo $LD_LIBRARY_PATH\n - name: Build extension for --load-extension test\n run: |-\n (cd tests && gcc ext.c -fPIC -shared -o ext.so)\n - name: Install dependencies\n run: |\n pip install . --group dev\n pip freeze\n - name: Run tests\n run: |\n pytest -n auto -m \"not serial\"\n pytest -m \"serial\"\n" }, { "path": ".github/workflows/test.yml", "content": "name: Test\n\non: [push, pull_request]\n\npermissions:\n contents: read\n\njobs:\n test:\n runs-on: ubuntu-latest\n strategy:\n matrix:\n python-version: [\"3.10\", \"3.11\", \"3.12\", \"3.13\", \"3.14\"]\n steps:\n - uses: actions/checkout@v4\n - name: Set up Python ${{ matrix.python-version }}\n uses: actions/setup-python@v6\n with:\n python-version: ${{ matrix.python-version }}\n allow-prereleases: true\n cache: pip\n cache-dependency-path: pyproject.toml\n - name: Build extension for --load-extension test\n run: |-\n (cd tests && gcc ext.c -fPIC -shared -o ext.so)\n - name: Install dependencies\n run: |\n pip install . --group dev\n pip freeze\n - name: Run tests\n run: |\n pytest -n auto -m \"not serial\"\n pytest -m \"serial\"\n # And the test that exceeds a localhost HTTPS server\n tests/test_datasette_https_server.sh\n - name: Black\n run: |\n black --version\n black --check .\n - name: Ruff\n run: ruff check datasette tests\n - name: Check if cog needs to be run\n run: |\n cog --check docs/*.rst\n - name: Check if blacken-docs needs to be run\n run: |\n # This fails on syntax errors, or a diff was applied\n blacken-docs -l 60 docs/*.rst\n - name: Test DATASETTE_LOAD_PLUGINS\n run: |\n pip install datasette-init datasette-json-html\n tests/test-datasette-load-plugins.sh\n" }, { "path": ".github/workflows/tmate-mac.yml", "content": "name: tmate session mac\n\non:\n workflow_dispatch:\n\npermissions:\n contents: read\n\njobs:\n build:\n runs-on: macos-latest\n steps:\n - uses: actions/checkout@v2\n - name: Setup tmate session\n uses: mxschmitt/action-tmate@v3\n" }, { "path": ".github/workflows/tmate.yml", "content": "name: tmate session\n\non:\n workflow_dispatch:\n\npermissions:\n contents: read\n models: read\n\njobs:\n build:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v2\n - name: Setup tmate session\n uses: mxschmitt/action-tmate@v3\n env:\n GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n" }, { "path": ".gitignore", "content": "build-metadata.json\ndatasets.json\n\nscratchpad\n\n.vscode\n\nuv.lock\ndata.db\n\n# test databases\n*.db\n\n# We don't use Pipfile, so ignore them\nPipfile\nPipfile.lock\n\nfixtures.db\n*test.db\n\n# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packaging\n.Python\nenv/\nbuild/\ndevelop-eggs/\ndist/\ndownloads/\neggs/\n.eggs/\nlib/\nlib64/\nparts/\nsdist/\nvar/\nwheels/\n*.egg-info/\n.installed.cfg\n*.egg\n\n# PyInstaller\n# Usually these files are written by a python script from a template\n# before PyInstaller builds the exe, so as to inject date/other infos into it.\n*.manifest\n*.spec\n\n# Installer logs\npip-log.txt\npip-delete-this-directory.txt\n\n# Unit test / coverage reports\nhtmlcov/\n.tox/\n.coverage\n.coverage.*\n.cache\nnosetests.xml\ncoverage.xml\n*.cover\n.hypothesis/\n\n# Translations\n*.mo\n*.pot\n\n# Django stuff:\n*.log\nlocal_settings.py\n\n# Flask stuff:\ninstance/\n.webassets-cache\n\n# Scrapy stuff:\n.scrapy\n\n# Sphinx documentation\ndocs/_build/\n\n# PyBuilder\ntarget/\n\n# Jupyter Notebook\n.ipynb_checkpoints\n\n# pyenv\n.python-version\n\n# celery beat schedule file\ncelerybeat-schedule\n\n# SageMath parsed files\n*.sage.py\n\n# dotenv\n.env\n\n# virtualenv\n.venv\nvenv/\nENV/\n\n# Spyder project settings\n.spyderproject\n.spyproject\n\n# Rope project settings\n.ropeproject\n\n# mkdocs documentation\n/site\n\n# mypy\n.mypy_cache/\n\n# macOS files\n.DS_Store\nnode_modules\n.*.swp\n\n# In case someone compiled tests/ext.c for test_load_extensions, don't\n# include it in source control.\ntests/*.dylib\ntests/*.so\ntests/*.dll\n\n.idea" }, { "path": ".isort.cfg", "content": "[settings]\nmulti_line_output=3\n\n" }, { "path": ".prettierrc", "content": "{\n \"tabWidth\": 2,\n \"useTabs\": false\n}\n" }, { "path": ".readthedocs.yaml", "content": "version: 2\n\nsphinx:\n configuration: docs/conf.py\n\nbuild:\n os: ubuntu-24.04\n tools:\n python: \"3.13\"\n jobs:\n install:\n - pip install --upgrade pip\n - pip install . --group dev\n\nformats:\n- pdf\n- epub\n" }, { "path": "CODE_OF_CONDUCT.md", "content": "# Contributor Covenant Code of Conduct\n\n## Our Pledge\n\nWe as members, contributors, and leaders pledge to make participation in our\ncommunity a harassment-free experience for everyone, regardless of age, body\nsize, visible or invisible disability, ethnicity, sex characteristics, gender\nidentity and expression, level of experience, education, socio-economic status,\nnationality, personal appearance, race, religion, or sexual identity\nand orientation.\n\nWe pledge to act and interact in ways that contribute to an open, welcoming,\ndiverse, inclusive, and healthy community.\n\n## Our Standards\n\nExamples of behavior that contributes to a positive environment for our\ncommunity include:\n\n* Demonstrating empathy and kindness toward other people\n* Being respectful of differing opinions, viewpoints, and experiences\n* Giving and gracefully accepting constructive feedback\n* Accepting responsibility and apologizing to those affected by our mistakes,\n and learning from the experience\n* Focusing on what is best not just for us as individuals, but for the\n overall community\n\nExamples of unacceptable behavior include:\n\n* The use of sexualized language or imagery, and sexual attention or\n advances of any kind\n* Trolling, insulting or derogatory comments, and personal or political attacks\n* Public or private harassment\n* Publishing others' private information, such as a physical or email\n address, without their explicit permission\n* Other conduct which could reasonably be considered inappropriate in a\n professional setting\n\n## Enforcement Responsibilities\n\nCommunity leaders are responsible for clarifying and enforcing our standards of\nacceptable behavior and will take appropriate and fair corrective action in\nresponse to any behavior that they deem inappropriate, threatening, offensive,\nor harmful.\n\nCommunity leaders have the right and responsibility to remove, edit, or reject\ncomments, commits, code, wiki edits, issues, and other contributions that are\nnot aligned to this Code of Conduct, and will communicate reasons for moderation\ndecisions when appropriate.\n\n## Scope\n\nThis Code of Conduct applies within all community spaces, and also applies when\nan individual is officially representing the community in public spaces.\nExamples of representing our community include using an official e-mail address,\nposting via an official social media account, or acting as an appointed\nrepresentative at an online or offline event.\n\n## Enforcement\n\nInstances of abusive, harassing, or otherwise unacceptable behavior may be\nreported to the community leaders responsible for enforcement at\n`swillison+datasette-code-of-conduct@gmail.com`.\nAll complaints will be reviewed and investigated promptly and fairly.\n\nAll community leaders are obligated to respect the privacy and security of the\nreporter of any incident.\n\n## Enforcement Guidelines\n\nCommunity leaders will follow these Community Impact Guidelines in determining\nthe consequences for any action they deem in violation of this Code of Conduct:\n\n### 1. Correction\n\n**Community Impact**: Use of inappropriate language or other behavior deemed\nunprofessional or unwelcome in the community.\n\n**Consequence**: A private, written warning from community leaders, providing\nclarity around the nature of the violation and an explanation of why the\nbehavior was inappropriate. A public apology may be requested.\n\n### 2. Warning\n\n**Community Impact**: A violation through a single incident or series\nof actions.\n\n**Consequence**: A warning with consequences for continued behavior. No\ninteraction with the people involved, including unsolicited interaction with\nthose enforcing the Code of Conduct, for a specified period of time. This\nincludes avoiding interactions in community spaces as well as external channels\nlike social media. Violating these terms may lead to a temporary or\npermanent ban.\n\n### 3. Temporary Ban\n\n**Community Impact**: A serious violation of community standards, including\nsustained inappropriate behavior.\n\n**Consequence**: A temporary ban from any sort of interaction or public\ncommunication with the community for a specified period of time. No public or\nprivate interaction with the people involved, including unsolicited interaction\nwith those enforcing the Code of Conduct, is allowed during this period.\nViolating these terms may lead to a permanent ban.\n\n### 4. Permanent Ban\n\n**Community Impact**: Demonstrating a pattern of violation of community\nstandards, including sustained inappropriate behavior, harassment of an\nindividual, or aggression toward or disparagement of classes of individuals.\n\n**Consequence**: A permanent ban from any sort of public interaction within\nthe community.\n\n## Attribution\n\nThis Code of Conduct is adapted from the [Contributor Covenant][homepage],\nversion 2.0, available at\nhttps://www.contributor-covenant.org/version/2/0/code_of_conduct.html.\n\nCommunity Impact Guidelines were inspired by [Mozilla's code of conduct\nenforcement ladder](https://github.com/mozilla/diversity).\n\n[homepage]: https://www.contributor-covenant.org\n\nFor answers to common questions about this code of conduct, see the FAQ at\nhttps://www.contributor-covenant.org/faq. Translations are available at\nhttps://www.contributor-covenant.org/translations.\n" }, { "path": "Dockerfile", "content": "FROM python:3.11.0-slim-bullseye as build\n\n# Version of Datasette to install, e.g. 0.55\n# docker build . -t datasette --build-arg VERSION=0.55\nARG VERSION\n\nRUN apt-get update && \\\n apt-get install -y --no-install-recommends libsqlite3-mod-spatialite && \\\n apt clean && \\\n rm -rf /var/lib/apt && \\\n rm -rf /var/lib/dpkg/info/*\n\nRUN pip install https://github.com/simonw/datasette/archive/refs/tags/${VERSION}.zip && \\\n find /usr/local/lib -name '__pycache__' | xargs rm -r && \\\n rm -rf /root/.cache/pip\n\nEXPOSE 8001\nCMD [\"datasette\"]\n" }, { "path": "Justfile", "content": "export DATASETTE_SECRET := \"not_a_secret\"\n\n# Run tests and linters\n@default: test lint\n\n# Setup project\n@init:\n uv sync\n\n# Run pytest with supplied options\n@test *options: init\n uv run pytest -n auto {{options}}\n\n@codespell:\n uv run codespell README.md --ignore-words docs/codespell-ignore-words.txt\n uv run codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt\n uv run codespell datasette -S datasette/static --ignore-words docs/codespell-ignore-words.txt\n uv run codespell tests --ignore-words docs/codespell-ignore-words.txt\n\n# Run linters: black, ruff, cog\n@lint: codespell\n uv run black datasette tests --check\n uv run ruff check datasette tests\n uv run cog --check README.md docs/*.rst\n\n# Apply ruff fixes\n@fix:\n uv run ruff check --fix datasette tests\n\n# Rebuild docs with cog\n@cog:\n uv run cog -r README.md docs/*.rst\n\n# Serve live docs on localhost:8000\n@docs: cog blacken-docs\n uv run make -C docs livehtml\n\n# Build docs as static HTML\n@docs-build: cog blacken-docs\n rm -rf docs/_build && cd docs && uv run make html\n\n# Apply Black\n@black:\n uv run black datasette tests\n\n# Apply blacken-docs\n@blacken-docs:\n uv run blacken-docs -l 60 docs/*.rst\n\n# Apply prettier\n@prettier:\n npm run fix\n\n# Format code with both black and prettier\n@format: black prettier blacken-docs\n\n@serve *options:\n uv run sqlite-utils create-database data.db\n uv run sqlite-utils create-table data.db docs id integer title text --pk id --ignore\n uv run python -m datasette data.db --root --reload {{options}}\n" }, { "path": "LICENSE", "content": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"{}\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright {yyyy} {name of copyright owner}\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, { "path": "MANIFEST.in", "content": "recursive-include datasette/static *\nrecursive-include datasette/templates *\ninclude versioneer.py\ninclude datasette/_version.py\ninclude LICENSE\n" }, { "path": "README.md", "content": "\"Datasette\"\n\n[![PyPI](https://img.shields.io/pypi/v/datasette.svg)](https://pypi.org/project/datasette/)\n[![Changelog](https://img.shields.io/github/v/release/simonw/datasette?label=changelog)](https://docs.datasette.io/en/latest/changelog.html)\n[![Python 3.x](https://img.shields.io/pypi/pyversions/datasette.svg?logo=python&logoColor=white)](https://pypi.org/project/datasette/)\n[![Tests](https://github.com/simonw/datasette/workflows/Test/badge.svg)](https://github.com/simonw/datasette/actions?query=workflow%3ATest)\n[![Documentation Status](https://readthedocs.org/projects/datasette/badge/?version=latest)](https://docs.datasette.io/en/latest/?badge=latest)\n[![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/datasette/blob/main/LICENSE)\n[![docker: datasette](https://img.shields.io/badge/docker-datasette-blue)](https://hub.docker.com/r/datasetteproject/datasette)\n[![discord](https://img.shields.io/discord/823971286308356157?label=discord)](https://datasette.io/discord)\n\n*An open source multi-tool for exploring and publishing data*\n\nDatasette is a tool for exploring and publishing data. It helps people take data of any shape or size and publish that as an interactive, explorable website and accompanying API.\n\nDatasette is aimed at data journalists, museum curators, archivists, local governments, scientists, researchers and anyone else who has data that they wish to share with the world.\n\n[Explore a demo](https://datasette.io/global-power-plants/global-power-plants), watch [a video about the project](https://simonwillison.net/2021/Feb/7/video/) or try it out [on GitHub Codespaces](https://github.com/datasette/datasette-studio).\n\n* [datasette.io](https://datasette.io/) is the official project website\n* Latest [Datasette News](https://datasette.io/news)\n* Comprehensive documentation: https://docs.datasette.io/\n* Examples: https://datasette.io/examples\n* Live demo of current `main` branch: https://latest.datasette.io/\n* Questions, feedback or want to talk about the project? Join our [Discord](https://datasette.io/discord)\n\nWant to stay up-to-date with the project? Subscribe to the [Datasette newsletter](https://datasette.substack.com/) for tips, tricks and news on what's new in the Datasette ecosystem.\n\n## Installation\n\nIf you are on a Mac, [Homebrew](https://brew.sh/) is the easiest way to install Datasette:\n\n brew install datasette\n\nYou can also install it using `pip` or `pipx`:\n\n pip install datasette\n\nDatasette requires Python 3.8 or higher. We also have [detailed installation instructions](https://docs.datasette.io/en/stable/installation.html) covering other options such as Docker.\n\n## Basic usage\n\n datasette serve path/to/database.db\n\nThis will start a web server on port 8001 - visit http://localhost:8001/ to access the web interface.\n\n`serve` is the default subcommand, you can omit it if you like.\n\nUse Chrome on OS X? You can run datasette against your browser history like so:\n\n datasette ~/Library/Application\\ Support/Google/Chrome/Default/History --nolock\n\nNow visiting http://localhost:8001/History/downloads will show you a web interface to browse your downloads data:\n\n![Downloads table rendered by datasette](https://static.simonwillison.net/static/2017/datasette-downloads.png)\n\n## metadata.json\n\nIf you want to include licensing and source information in the generated datasette website you can do so using a JSON file that looks something like this:\n\n {\n \"title\": \"Five Thirty Eight\",\n \"license\": \"CC Attribution 4.0 License\",\n \"license_url\": \"http://creativecommons.org/licenses/by/4.0/\",\n \"source\": \"fivethirtyeight/data on GitHub\",\n \"source_url\": \"https://github.com/fivethirtyeight/data\"\n }\n\nSave this in `metadata.json` and run Datasette like so:\n\n datasette serve fivethirtyeight.db -m metadata.json\n\nThe license and source information will be displayed on the index page and in the footer. They will also be included in the JSON produced by the API.\n\n## datasette publish\n\nIf you have [Heroku](https://heroku.com/) or [Google Cloud Run](https://cloud.google.com/run/) configured, Datasette can deploy one or more SQLite databases to the internet with a single command:\n\n datasette publish heroku database.db\n\nOr:\n\n datasette publish cloudrun database.db\n\nThis will create a docker image containing both the datasette application and the specified SQLite database files. It will then deploy that image to Heroku or Cloud Run and give you a URL to access the resulting website and API.\n\nSee [Publishing data](https://docs.datasette.io/en/stable/publish.html) in the documentation for more details.\n\n## Datasette Lite\n\n[Datasette Lite](https://lite.datasette.io/) is Datasette packaged using WebAssembly so that it runs entirely in your browser, no Python web application server required. Read more about that in the [Datasette Lite documentation](https://github.com/simonw/datasette-lite/blob/main/README.md).\n" }, { "path": "codecov.yml", "content": "coverage:\n status:\n project:\n default:\n informational: true\n patch:\n default:\n informational: true\n" }, { "path": "datasette/__init__.py", "content": "from datasette.permissions import Permission # noqa\nfrom datasette.version import __version_info__, __version__ # noqa\nfrom datasette.events import Event # noqa\nfrom datasette.tokens import TokenHandler, TokenRestrictions # noqa\nfrom datasette.utils.asgi import Forbidden, NotFound, Request, Response # noqa\nfrom datasette.utils import actor_matches_allow # noqa\nfrom datasette.views import Context # noqa\nfrom .hookspecs import hookimpl # noqa\nfrom .hookspecs import hookspec # noqa\n" }, { "path": "datasette/__main__.py", "content": "from datasette.cli import cli\n\nif __name__ == \"__main__\":\n cli()\n" }, { "path": "datasette/actor_auth_cookie.py", "content": "from datasette import hookimpl\nfrom itsdangerous import BadSignature\nfrom datasette.utils import baseconv\nimport time\n\n\n@hookimpl\ndef actor_from_request(datasette, request):\n if \"ds_actor\" not in request.cookies:\n return None\n try:\n decoded = datasette.unsign(request.cookies[\"ds_actor\"], \"actor\")\n # If it has \"e\" and \"a\" keys process the \"e\" expiry\n if not isinstance(decoded, dict) or \"a\" not in decoded:\n return None\n expires_at = decoded.get(\"e\")\n if expires_at:\n timestamp = int(baseconv.base62.decode(expires_at))\n if time.time() > timestamp:\n return None\n return decoded[\"a\"]\n except BadSignature:\n return None\n" }, { "path": "datasette/app.py", "content": "from __future__ import annotations\n\nfrom asgi_csrf import Errors\nimport asyncio\nimport contextvars\nfrom typing import TYPE_CHECKING, Any, Dict, Iterable, List\n\nif TYPE_CHECKING:\n from datasette.permissions import Resource\n from datasette.tokens import TokenRestrictions\nimport asgi_csrf\nimport collections\nimport dataclasses\nimport datetime\nimport functools\nimport glob\nimport hashlib\nimport httpx\nimport importlib.metadata\nimport inspect\nfrom itsdangerous import BadSignature\nimport json\nimport os\nimport re\nimport secrets\nimport sys\nimport threading\nimport time\nimport types\nimport urllib.parse\nfrom concurrent import futures\nfrom pathlib import Path\n\nfrom markupsafe import Markup, escape\nfrom itsdangerous import URLSafeSerializer\nfrom jinja2 import (\n ChoiceLoader,\n Environment,\n FileSystemLoader,\n PrefixLoader,\n)\nfrom jinja2.environment import Template\nfrom jinja2.exceptions import TemplateNotFound\n\nfrom .events import Event\nfrom .column_types import SQLiteType\nfrom .views import Context\nfrom .views.database import database_download, DatabaseView, TableCreateView, QueryView\nfrom .views.index import IndexView\nfrom .views.special import (\n JsonDataView,\n PatternPortfolioView,\n AuthTokenView,\n ApiExplorerView,\n CreateTokenView,\n LogoutView,\n AllowDebugView,\n PermissionsDebugView,\n MessagesDebugView,\n AllowedResourcesView,\n PermissionRulesView,\n PermissionCheckView,\n TablesView,\n InstanceSchemaView,\n DatabaseSchemaView,\n TableSchemaView,\n)\nfrom .views.table import (\n TableInsertView,\n TableUpsertView,\n TableSetColumnTypeView,\n TableDropView,\n table_view,\n)\nfrom .views.row import RowView, RowDeleteView, RowUpdateView\nfrom .renderer import json_renderer\nfrom .url_builder import Urls\nfrom .database import Database, QueryInterrupted\n\nfrom .utils import (\n PaginatedResources,\n PrefixedUrlString,\n SPATIALITE_FUNCTIONS,\n StartupError,\n async_call_with_supported_arguments,\n await_me_maybe,\n baseconv,\n call_with_supported_arguments,\n detect_json1,\n display_actor,\n escape_css_string,\n escape_sqlite,\n find_spatialite,\n format_bytes,\n module_from_path,\n move_plugins_and_allow,\n move_table_config,\n parse_metadata,\n resolve_env_secrets,\n resolve_routes,\n tilde_decode,\n tilde_encode,\n to_css_class,\n urlsafe_components,\n redact_keys,\n row_sql_params_pks,\n)\nfrom .utils.asgi import (\n AsgiLifespan,\n Forbidden,\n NotFound,\n DatabaseNotFound,\n TableNotFound,\n RowNotFound,\n Request,\n Response,\n AsgiRunOnFirstRequest,\n asgi_static,\n asgi_send,\n asgi_send_file,\n asgi_send_redirect,\n)\nfrom .utils.internal_db import init_internal_db, populate_schema_tables\nfrom .utils.sqlite import (\n sqlite3,\n using_pysqlite3,\n)\nfrom .tracer import AsgiTracer\nfrom .plugins import pm, DEFAULT_PLUGINS, get_plugins\nfrom .version import __version__\n\nfrom .resources import DatabaseResource, TableResource\n\napp_root = Path(__file__).parent.parent\n\n\n# Context variable to track when code is executing within a datasette.client request\n_in_datasette_client = contextvars.ContextVar(\"in_datasette_client\", default=False)\n\n\nclass _DatasetteClientContext:\n \"\"\"Context manager to mark code as executing within a datasette.client request.\"\"\"\n\n def __enter__(self):\n self.token = _in_datasette_client.set(True)\n return self\n\n def __exit__(self, exc_type, exc_val, exc_tb):\n _in_datasette_client.reset(self.token)\n return False\n\n\n@dataclasses.dataclass\nclass PermissionCheck:\n \"\"\"Represents a logged permission check for debugging purposes.\"\"\"\n\n when: str\n actor: Dict[str, Any] | None\n action: str\n parent: str | None\n child: str | None\n result: bool\n\n\n# https://github.com/simonw/datasette/issues/283#issuecomment-781591015\nSQLITE_LIMIT_ATTACHED = 10\n\nINTERNAL_DB_NAME = \"__INTERNAL__\"\n\nSetting = collections.namedtuple(\"Setting\", (\"name\", \"default\", \"help\"))\nSETTINGS = (\n Setting(\"default_page_size\", 100, \"Default page size for the table view\"),\n Setting(\n \"max_returned_rows\",\n 1000,\n \"Maximum rows that can be returned from a table or custom query\",\n ),\n Setting(\n \"max_insert_rows\",\n 100,\n \"Maximum rows that can be inserted at a time using the bulk insert API\",\n ),\n Setting(\n \"num_sql_threads\",\n 3,\n \"Number of threads in the thread pool for executing SQLite queries\",\n ),\n Setting(\"sql_time_limit_ms\", 1000, \"Time limit for a SQL query in milliseconds\"),\n Setting(\n \"default_facet_size\", 30, \"Number of values to return for requested facets\"\n ),\n Setting(\"facet_time_limit_ms\", 200, \"Time limit for calculating a requested facet\"),\n Setting(\n \"facet_suggest_time_limit_ms\",\n 50,\n \"Time limit for calculating a suggested facet\",\n ),\n Setting(\n \"allow_facet\",\n True,\n \"Allow users to specify columns to facet using ?_facet= parameter\",\n ),\n Setting(\n \"allow_download\",\n True,\n \"Allow users to download the original SQLite database files\",\n ),\n Setting(\n \"allow_signed_tokens\",\n True,\n \"Allow users to create and use signed API tokens\",\n ),\n Setting(\n \"default_allow_sql\",\n True,\n \"Allow anyone to run arbitrary SQL queries\",\n ),\n Setting(\n \"max_signed_tokens_ttl\",\n 0,\n \"Maximum allowed expiry time for signed API tokens\",\n ),\n Setting(\"suggest_facets\", True, \"Calculate and display suggested facets\"),\n Setting(\n \"default_cache_ttl\",\n 5,\n \"Default HTTP cache TTL (used in Cache-Control: max-age= header)\",\n ),\n Setting(\"cache_size_kb\", 0, \"SQLite cache size in KB (0 == use SQLite default)\"),\n Setting(\n \"allow_csv_stream\",\n True,\n \"Allow .csv?_stream=1 to download all rows (ignoring max_returned_rows)\",\n ),\n Setting(\n \"max_csv_mb\",\n 100,\n \"Maximum size allowed for CSV export in MB - set 0 to disable this limit\",\n ),\n Setting(\n \"truncate_cells_html\",\n 2048,\n \"Truncate cells longer than this in HTML table view - set 0 to disable\",\n ),\n Setting(\n \"force_https_urls\",\n False,\n \"Force URLs in API output to always use https:// protocol\",\n ),\n Setting(\n \"template_debug\",\n False,\n \"Allow display of template debug information with ?_context=1\",\n ),\n Setting(\n \"trace_debug\",\n False,\n \"Allow display of SQL trace debug information with ?_trace=1\",\n ),\n Setting(\"base_url\", \"/\", \"Datasette URLs should use this base path\"),\n)\n_HASH_URLS_REMOVED = \"The hash_urls setting has been removed, try the datasette-hashed-urls plugin instead\"\nOBSOLETE_SETTINGS = {\n \"hash_urls\": _HASH_URLS_REMOVED,\n \"default_cache_ttl_hashed\": _HASH_URLS_REMOVED,\n}\nDEFAULT_SETTINGS = {option.name: option.default for option in SETTINGS}\n\nFAVICON_PATH = app_root / \"datasette\" / \"static\" / \"favicon.png\"\n\nDEFAULT_NOT_SET = object()\n\n\nResourcesSQL = collections.namedtuple(\"ResourcesSQL\", (\"sql\", \"params\"))\n\n\nasync def favicon(request, send):\n await asgi_send_file(\n send,\n str(FAVICON_PATH),\n content_type=\"image/png\",\n headers={\"Cache-Control\": \"max-age=3600, immutable, public\"},\n )\n\n\nResolvedTable = collections.namedtuple(\"ResolvedTable\", (\"db\", \"table\", \"is_view\"))\nResolvedRow = collections.namedtuple(\n \"ResolvedRow\", (\"db\", \"table\", \"sql\", \"params\", \"pks\", \"pk_values\", \"row\")\n)\n\n\ndef _to_string(value):\n if isinstance(value, str):\n return value\n else:\n return json.dumps(value, default=str)\n\n\nclass Datasette:\n # Message constants:\n INFO = 1\n WARNING = 2\n ERROR = 3\n\n def __init__(\n self,\n files=None,\n immutables=None,\n cache_headers=True,\n cors=False,\n inspect_data=None,\n config=None,\n metadata=None,\n sqlite_extensions=None,\n template_dir=None,\n plugins_dir=None,\n static_mounts=None,\n memory=False,\n settings=None,\n secret=None,\n version_note=None,\n config_dir=None,\n pdb=False,\n crossdb=False,\n nolock=False,\n internal=None,\n default_deny=False,\n ):\n self._startup_invoked = False\n assert config_dir is None or isinstance(\n config_dir, Path\n ), \"config_dir= should be a pathlib.Path\"\n self.config_dir = config_dir\n self.pdb = pdb\n self._secret = secret or secrets.token_hex(32)\n if files is not None and isinstance(files, str):\n raise ValueError(\"files= must be a list of paths, not a string\")\n self.files = tuple(files or []) + tuple(immutables or [])\n if config_dir:\n db_files = []\n for ext in (\"db\", \"sqlite\", \"sqlite3\"):\n db_files.extend(config_dir.glob(\"*.{}\".format(ext)))\n self.files += tuple(str(f) for f in db_files)\n if (\n config_dir\n and (config_dir / \"inspect-data.json\").exists()\n and not inspect_data\n ):\n inspect_data = json.loads((config_dir / \"inspect-data.json\").read_text())\n if not immutables:\n immutable_filenames = [i[\"file\"] for i in inspect_data.values()]\n immutables = [\n f for f in self.files if Path(f).name in immutable_filenames\n ]\n self.inspect_data = inspect_data\n self.immutables = set(immutables or [])\n self.databases = collections.OrderedDict()\n self.actions = {} # .invoke_startup() will populate this\n self._column_types = {} # .invoke_startup() will populate this\n try:\n self._refresh_schemas_lock = asyncio.Lock()\n except RuntimeError as rex:\n # Workaround for intermittent test failure, see:\n # https://github.com/simonw/datasette/issues/1802\n if \"There is no current event loop in thread\" in str(rex):\n loop = asyncio.new_event_loop()\n asyncio.set_event_loop(loop)\n self._refresh_schemas_lock = asyncio.Lock()\n else:\n raise\n self.crossdb = crossdb\n self.nolock = nolock\n if memory or crossdb or not self.files:\n self.add_database(\n Database(self, is_mutable=False, is_memory=True), name=\"_memory\"\n )\n for file in self.files:\n self.add_database(\n Database(self, file, is_mutable=file not in self.immutables)\n )\n\n self.internal_db_created = False\n if internal is None:\n self._internal_database = Database(self, memory_name=secrets.token_hex())\n else:\n self._internal_database = Database(self, path=internal, mode=\"rwc\")\n self._internal_database.name = INTERNAL_DB_NAME\n\n self.cache_headers = cache_headers\n self.cors = cors\n config_files = []\n metadata_files = []\n if config_dir:\n metadata_files = [\n config_dir / filename\n for filename in (\"metadata.json\", \"metadata.yaml\", \"metadata.yml\")\n if (config_dir / filename).exists()\n ]\n config_files = [\n config_dir / filename\n for filename in (\"datasette.json\", \"datasette.yaml\", \"datasette.yml\")\n if (config_dir / filename).exists()\n ]\n if config_dir and metadata_files and not metadata:\n with metadata_files[0].open() as fp:\n metadata = parse_metadata(fp.read())\n\n if config_dir and config_files and not config:\n with config_files[0].open() as fp:\n config = parse_metadata(fp.read())\n\n # Move any \"plugins\" and \"allow\" settings from metadata to config - updates them in place\n metadata = metadata or {}\n config = config or {}\n metadata, config = move_plugins_and_allow(metadata, config)\n # Now migrate any known table configuration settings over as well\n metadata, config = move_table_config(metadata, config)\n\n self._metadata_local = metadata or {}\n self.sqlite_extensions = []\n for extension in sqlite_extensions or []:\n # Resolve spatialite, if requested\n if extension == \"spatialite\":\n # Could raise SpatialiteNotFound\n self.sqlite_extensions.append(find_spatialite())\n else:\n self.sqlite_extensions.append(extension)\n if config_dir and (config_dir / \"templates\").is_dir() and not template_dir:\n template_dir = str((config_dir / \"templates\").resolve())\n self.template_dir = template_dir\n if config_dir and (config_dir / \"plugins\").is_dir() and not plugins_dir:\n plugins_dir = str((config_dir / \"plugins\").resolve())\n self.plugins_dir = plugins_dir\n if config_dir and (config_dir / \"static\").is_dir() and not static_mounts:\n static_mounts = [(\"static\", str((config_dir / \"static\").resolve()))]\n self.static_mounts = static_mounts or []\n if config_dir and (config_dir / \"datasette.json\").exists() and not config:\n config = json.loads((config_dir / \"datasette.json\").read_text())\n\n config = config or {}\n config_settings = config.get(\"settings\") or {}\n\n # Validate settings from config file\n for key, value in config_settings.items():\n if key not in DEFAULT_SETTINGS:\n raise StartupError(f\"Invalid setting '{key}' in config file\")\n # Validate type matches expected type from DEFAULT_SETTINGS\n if value is not None: # Allow None/null values\n expected_type = type(DEFAULT_SETTINGS[key])\n actual_type = type(value)\n if actual_type != expected_type:\n raise StartupError(\n f\"Setting '{key}' in config file has incorrect type. \"\n f\"Expected {expected_type.__name__}, got {actual_type.__name__}. \"\n f\"Value: {value!r}. \"\n f\"Hint: In YAML/JSON config files, remove quotes from boolean and integer values.\"\n )\n\n # Validate settings from constructor parameter\n if settings:\n for key, value in settings.items():\n if key not in DEFAULT_SETTINGS:\n raise StartupError(f\"Invalid setting '{key}' in settings parameter\")\n if value is not None:\n expected_type = type(DEFAULT_SETTINGS[key])\n actual_type = type(value)\n if actual_type != expected_type:\n raise StartupError(\n f\"Setting '{key}' in settings parameter has incorrect type. \"\n f\"Expected {expected_type.__name__}, got {actual_type.__name__}. \"\n f\"Value: {value!r}\"\n )\n\n self.config = config\n # CLI settings should overwrite datasette.json settings\n self._settings = dict(DEFAULT_SETTINGS, **(config_settings), **(settings or {}))\n self.renderers = {} # File extension -> (renderer, can_render) functions\n self.version_note = version_note\n if self.setting(\"num_sql_threads\") == 0:\n self.executor = None\n else:\n self.executor = futures.ThreadPoolExecutor(\n max_workers=self.setting(\"num_sql_threads\")\n )\n self.max_returned_rows = self.setting(\"max_returned_rows\")\n self.sql_time_limit_ms = self.setting(\"sql_time_limit_ms\")\n self.page_size = self.setting(\"default_page_size\")\n # Execute plugins in constructor, to ensure they are available\n # when the rest of `datasette inspect` executes\n if self.plugins_dir:\n for filepath in glob.glob(os.path.join(self.plugins_dir, \"*.py\")):\n if not os.path.isfile(filepath):\n continue\n mod = module_from_path(filepath, name=os.path.basename(filepath))\n try:\n pm.register(mod)\n except ValueError:\n # Plugin already registered\n pass\n\n # Configure Jinja\n default_templates = str(app_root / \"datasette\" / \"templates\")\n template_paths = []\n if self.template_dir:\n template_paths.append(self.template_dir)\n plugin_template_paths = [\n plugin[\"templates_path\"]\n for plugin in get_plugins()\n if plugin[\"templates_path\"]\n ]\n template_paths.extend(plugin_template_paths)\n template_paths.append(default_templates)\n template_loader = ChoiceLoader(\n [\n FileSystemLoader(template_paths),\n # Support {% extends \"default:table.html\" %}:\n PrefixLoader(\n {\"default\": FileSystemLoader(default_templates)}, delimiter=\":\"\n ),\n ]\n )\n environment = Environment(\n loader=template_loader,\n autoescape=True,\n enable_async=True,\n # undefined=StrictUndefined,\n )\n environment.filters[\"escape_css_string\"] = escape_css_string\n environment.filters[\"quote_plus\"] = urllib.parse.quote_plus\n self._jinja_env = environment\n environment.filters[\"escape_sqlite\"] = escape_sqlite\n environment.filters[\"to_css_class\"] = to_css_class\n self._register_renderers()\n self._permission_checks = collections.deque(maxlen=200)\n self._root_token = secrets.token_hex(32)\n self.root_enabled = False\n self.default_deny = default_deny\n self.client = DatasetteClient(self)\n\n async def apply_metadata_json(self):\n # Apply any metadata entries from metadata.json to the internal tables\n # step 1: top-level metadata\n for key in self._metadata_local or {}:\n if key == \"databases\":\n continue\n value = self._metadata_local[key]\n await self.set_instance_metadata(key, _to_string(value))\n\n # step 2: database-level metadata\n for dbname, db in self._metadata_local.get(\"databases\", {}).items():\n for key, value in db.items():\n if key in (\"tables\", \"queries\"):\n continue\n await self.set_database_metadata(dbname, key, _to_string(value))\n\n # step 3: table-level metadata\n for tablename, table in db.get(\"tables\", {}).items():\n for key, value in table.items():\n if key == \"columns\":\n continue\n await self.set_resource_metadata(\n dbname, tablename, key, _to_string(value)\n )\n\n # step 4: column-level metadata (only descriptions in metadata.json)\n for columnname, column_description in table.get(\"columns\", {}).items():\n await self.set_column_metadata(\n dbname, tablename, columnname, \"description\", column_description\n )\n\n # TODO(alex) is metadata.json was loaded in, and --internal is not memory, then log\n # a warning to user that they should delete their metadata.json file\n\n def get_jinja_environment(self, request: Request = None) -> Environment:\n environment = self._jinja_env\n if request:\n for environment in pm.hook.jinja2_environment_from_request(\n datasette=self, request=request, env=environment\n ):\n pass\n return environment\n\n def get_action(self, name_or_abbr: str):\n \"\"\"\n Returns an Action object for the given name or abbreviation. Returns None if not found.\n \"\"\"\n if name_or_abbr in self.actions:\n return self.actions[name_or_abbr]\n # Try abbreviation\n for action in self.actions.values():\n if action.abbr == name_or_abbr:\n return action\n return None\n\n async def refresh_schemas(self):\n # Throttle schema refreshes to at most once per second\n if time.monotonic() - getattr(self, \"_last_schema_refresh\", 0) < 1.0:\n return\n self._last_schema_refresh = time.monotonic()\n if self._refresh_schemas_lock.locked():\n return\n async with self._refresh_schemas_lock:\n await self._refresh_schemas()\n\n async def _refresh_schemas(self):\n internal_db = self.get_internal_database()\n if not self.internal_db_created:\n await init_internal_db(internal_db)\n await self.apply_metadata_json()\n self.internal_db_created = True\n current_schema_versions = {\n row[\"database_name\"]: row[\"schema_version\"]\n for row in await internal_db.execute(\n \"select database_name, schema_version from catalog_databases\"\n )\n }\n # Delete stale entries for databases that are no longer attached\n stale_databases = set(current_schema_versions.keys()) - set(\n self.databases.keys()\n )\n for stale_db_name in stale_databases:\n await internal_db.execute_write(\n \"DELETE FROM catalog_databases WHERE database_name = ?\",\n [stale_db_name],\n )\n for database_name, db in self.databases.items():\n schema_version = (await db.execute(\"PRAGMA schema_version\")).first()[0]\n # Compare schema versions to see if we should skip it\n if schema_version == current_schema_versions.get(database_name):\n continue\n placeholders = \"(?, ?, ?, ?)\"\n values = [database_name, str(db.path), db.is_memory, schema_version]\n if db.path is None:\n placeholders = \"(?, null, ?, ?)\"\n values = [database_name, db.is_memory, schema_version]\n await internal_db.execute_write(\n \"\"\"\n INSERT OR REPLACE INTO catalog_databases (database_name, path, is_memory, schema_version)\n VALUES {}\n \"\"\".format(placeholders),\n values,\n )\n await populate_schema_tables(internal_db, db)\n\n @property\n def urls(self):\n return Urls(self)\n\n @property\n def pm(self):\n \"\"\"\n Return the global plugin manager instance.\n\n This provides access to the pluggy PluginManager that manages all\n Datasette plugins and hooks. Use datasette.pm.hook.hook_name() to\n call plugin hooks.\n \"\"\"\n return pm\n\n async def invoke_startup(self):\n # This must be called for Datasette to be in a usable state\n if self._startup_invoked:\n return\n # Register event classes\n event_classes = []\n for hook in pm.hook.register_events(datasette=self):\n extra_classes = await await_me_maybe(hook)\n if extra_classes:\n event_classes.extend(extra_classes)\n self.event_classes = tuple(event_classes)\n\n # Register actions, but watch out for duplicate name/abbr\n action_names = {}\n action_abbrs = {}\n for hook in pm.hook.register_actions(datasette=self):\n if hook:\n for action in hook:\n if (\n action.name in action_names\n and action != action_names[action.name]\n ):\n raise StartupError(\n \"Duplicate action name: {}\".format(action.name)\n )\n if (\n action.abbr\n and action.abbr in action_abbrs\n and action != action_abbrs[action.abbr]\n ):\n raise StartupError(\n \"Duplicate action abbr: {}\".format(action.abbr)\n )\n action_names[action.name] = action\n if action.abbr:\n action_abbrs[action.abbr] = action\n self.actions[action.name] = action\n\n # Register column types (classes, not instances)\n self._column_types = {}\n for hook in pm.hook.register_column_types(datasette=self):\n if hook:\n for ct_cls in hook:\n if ct_cls.name in self._column_types:\n raise StartupError(f\"Duplicate column type name: {ct_cls.name}\")\n self._column_types[ct_cls.name] = ct_cls\n\n for hook in pm.hook.prepare_jinja2_environment(\n env=self._jinja_env, datasette=self\n ):\n await await_me_maybe(hook)\n # Ensure internal tables and metadata are populated before startup hooks\n await self._refresh_schemas()\n # Load column_types from config into internal DB\n await self._apply_column_types_config()\n for hook in pm.hook.startup(datasette=self):\n await await_me_maybe(hook)\n self._startup_invoked = True\n\n def sign(self, value, namespace=\"default\"):\n return URLSafeSerializer(self._secret, namespace).dumps(value)\n\n def unsign(self, signed, namespace=\"default\"):\n return URLSafeSerializer(self._secret, namespace).loads(signed)\n\n def in_client(self) -> bool:\n \"\"\"Check if the current code is executing within a datasette.client request.\n\n Returns:\n bool: True if currently executing within a datasette.client request, False otherwise.\n \"\"\"\n return _in_datasette_client.get()\n\n def _token_handlers(self):\n \"\"\"Collect all registered token handlers from plugins.\"\"\"\n from datasette.tokens import TokenHandler\n\n handlers = []\n for result in pm.hook.register_token_handler(datasette=self):\n if isinstance(result, TokenHandler):\n handlers.append(result)\n elif isinstance(result, list):\n handlers.extend(h for h in result if isinstance(h, TokenHandler))\n return handlers\n\n async def create_token(\n self,\n actor_id: str,\n *,\n expires_after: int | None = None,\n restrictions: \"TokenRestrictions | None\" = None,\n handler: str | None = None,\n ) -> str:\n \"\"\"\n Create an API token for the given actor.\n\n Uses the first registered token handler by default, or a specific\n handler if ``handler`` is provided (matched by handler name).\n\n Pass a :class:`TokenRestrictions` to limit which actions the token\n can perform.\n \"\"\"\n handlers = self._token_handlers()\n if not handlers:\n raise RuntimeError(\"No token handlers are registered\")\n\n if handler is not None:\n matched = [h for h in handlers if h.name == handler]\n if not matched:\n available = [h.name for h in handlers]\n raise ValueError(\n f\"Token handler {handler!r} not found. \"\n f\"Available handlers: {available}\"\n )\n chosen = matched[0]\n else:\n chosen = handlers[0]\n\n return await chosen.create_token(\n self,\n actor_id,\n expires_after=expires_after,\n restrictions=restrictions,\n )\n\n async def verify_token(self, token: str) -> dict | None:\n \"\"\"\n Verify an API token by trying all registered token handlers.\n\n Returns an actor dict from the first handler that recognizes the\n token, or None if no handler accepts it.\n \"\"\"\n for token_handler in self._token_handlers():\n result = await token_handler.verify_token(self, token)\n if result is not None:\n return result\n return None\n\n def get_database(self, name=None, route=None):\n if route is not None:\n matches = [db for db in self.databases.values() if db.route == route]\n if not matches:\n raise KeyError\n return matches[0]\n if name is None:\n name = [key for key in self.databases.keys()][0]\n return self.databases[name]\n\n def add_database(self, db, name=None, route=None):\n new_databases = self.databases.copy()\n if name is None:\n # Pick a unique name for this database\n suggestion = db.suggest_name()\n name = suggestion\n else:\n suggestion = name\n i = 2\n while name in self.databases:\n name = \"{}_{}\".format(suggestion, i)\n i += 1\n db.name = name\n db.route = route or name\n new_databases[name] = db\n # don't mutate! that causes race conditions with live import\n self.databases = new_databases\n return db\n\n def add_memory_database(self, memory_name, name=None, route=None):\n return self.add_database(\n Database(self, memory_name=memory_name), name=name, route=route\n )\n\n def remove_database(self, name):\n self.get_database(name).close()\n new_databases = self.databases.copy()\n new_databases.pop(name)\n self.databases = new_databases\n\n def setting(self, key):\n return self._settings.get(key, None)\n\n def settings_dict(self):\n # Returns a fully resolved settings dictionary, useful for templates\n return {option.name: self.setting(option.name) for option in SETTINGS}\n\n def _metadata_recursive_update(self, orig, updated):\n if not isinstance(orig, dict) or not isinstance(updated, dict):\n return orig\n\n for key, upd_value in updated.items():\n if isinstance(upd_value, dict) and isinstance(orig.get(key), dict):\n orig[key] = self._metadata_recursive_update(orig[key], upd_value)\n else:\n orig[key] = upd_value\n return orig\n\n async def get_instance_metadata(self):\n rows = await self.get_internal_database().execute(\"\"\"\n SELECT\n key,\n value\n FROM metadata_instance\n \"\"\")\n return dict(rows)\n\n async def get_database_metadata(self, database_name: str):\n rows = await self.get_internal_database().execute(\n \"\"\"\n SELECT\n key,\n value\n FROM metadata_databases\n WHERE database_name = ?\n \"\"\",\n [database_name],\n )\n return dict(rows)\n\n async def get_resource_metadata(self, database_name: str, resource_name: str):\n rows = await self.get_internal_database().execute(\n \"\"\"\n SELECT\n key,\n value\n FROM metadata_resources\n WHERE database_name = ?\n AND resource_name = ?\n \"\"\",\n [database_name, resource_name],\n )\n return dict(rows)\n\n async def get_column_metadata(\n self, database_name: str, resource_name: str, column_name: str\n ):\n rows = await self.get_internal_database().execute(\n \"\"\"\n SELECT\n key,\n value\n FROM metadata_columns\n WHERE database_name = ?\n AND resource_name = ?\n AND column_name = ?\n \"\"\",\n [database_name, resource_name, column_name],\n )\n return dict(rows)\n\n async def set_instance_metadata(self, key: str, value: str):\n # TODO upsert only supported on SQLite 3.24.0 (2018-06-04)\n await self.get_internal_database().execute_write(\n \"\"\"\n INSERT INTO metadata_instance(key, value)\n VALUES(?, ?)\n ON CONFLICT(key) DO UPDATE SET value = excluded.value;\n \"\"\",\n [key, value],\n )\n\n async def set_database_metadata(self, database_name: str, key: str, value: str):\n # TODO upsert only supported on SQLite 3.24.0 (2018-06-04)\n await self.get_internal_database().execute_write(\n \"\"\"\n INSERT INTO metadata_databases(database_name, key, value)\n VALUES(?, ?, ?)\n ON CONFLICT(database_name, key) DO UPDATE SET value = excluded.value;\n \"\"\",\n [database_name, key, value],\n )\n\n async def set_resource_metadata(\n self, database_name: str, resource_name: str, key: str, value: str\n ):\n # TODO upsert only supported on SQLite 3.24.0 (2018-06-04)\n await self.get_internal_database().execute_write(\n \"\"\"\n INSERT INTO metadata_resources(database_name, resource_name, key, value)\n VALUES(?, ?, ?, ?)\n ON CONFLICT(database_name, resource_name, key) DO UPDATE SET value = excluded.value;\n \"\"\",\n [database_name, resource_name, key, value],\n )\n\n async def set_column_metadata(\n self,\n database_name: str,\n resource_name: str,\n column_name: str,\n key: str,\n value: str,\n ):\n # TODO upsert only supported on SQLite 3.24.0 (2018-06-04)\n await self.get_internal_database().execute_write(\n \"\"\"\n INSERT INTO metadata_columns(database_name, resource_name, column_name, key, value)\n VALUES(?, ?, ?, ?, ?)\n ON CONFLICT(database_name, resource_name, column_name, key) DO UPDATE SET value = excluded.value;\n \"\"\",\n [database_name, resource_name, column_name, key, value],\n )\n\n # Column types API\n\n async def _get_resource_column_details(self, database: str, resource: str):\n db = self.databases.get(database)\n if db is None:\n return {}\n try:\n return {\n column.name: column\n for column in await db.table_column_details(resource)\n }\n except sqlite3.OperationalError:\n return {}\n\n @staticmethod\n def _column_type_is_applicable(ct_cls, column_detail) -> bool:\n sqlite_types = getattr(ct_cls, \"sqlite_types\", None)\n if sqlite_types is None:\n return True\n if column_detail is None:\n return False\n actual_sqlite_type = SQLiteType.from_declared_type(column_detail.type)\n return actual_sqlite_type in sqlite_types\n\n async def _validate_column_type_assignment(\n self, database: str, resource: str, column: str, ct_cls\n ) -> None:\n sqlite_types = getattr(ct_cls, \"sqlite_types\", None)\n if sqlite_types is None:\n return\n\n column_detail = (\n await self._get_resource_column_details(database, resource)\n ).get(column)\n if column_detail is None:\n return\n\n actual_sqlite_type = SQLiteType.from_declared_type(column_detail.type)\n if actual_sqlite_type in sqlite_types:\n return\n\n allowed = \", \".join(sqlite_type.value for sqlite_type in sqlite_types)\n actual = (\n actual_sqlite_type.value\n if actual_sqlite_type is not None\n else \"unrecognized {!r}\".format(column_detail.type)\n )\n raise ValueError(\n \"Column type {!r} is only applicable to SQLite types {} but {}.{}.{} \"\n \"has SQLite type {}\".format(\n ct_cls.name,\n allowed,\n database,\n resource,\n column,\n actual,\n )\n )\n\n async def _apply_column_types_config(self):\n \"\"\"Load column_types from datasette.json config into the internal DB.\"\"\"\n import logging\n\n for db_name, db_conf in (self.config or {}).get(\"databases\", {}).items():\n for table_name, table_conf in db_conf.get(\"tables\", {}).items():\n for col_name, ct in table_conf.get(\"column_types\", {}).items():\n if isinstance(ct, str):\n col_type, config = ct, None\n else:\n col_type = ct[\"type\"]\n config = ct.get(\"config\")\n if col_type not in self._column_types:\n logging.warning(\n \"column_types config references unknown type %r \"\n \"for %s.%s.%s\",\n col_type,\n db_name,\n table_name,\n col_name,\n )\n try:\n await self.set_column_type(\n db_name, table_name, col_name, col_type, config\n )\n except ValueError as ex:\n logging.warning(str(ex))\n\n async def get_column_type(self, database: str, resource: str, column: str):\n \"\"\"\n Return a ColumnType instance (with config baked in) for a specific\n column, or None if no column type is assigned.\n \"\"\"\n row = await self.get_internal_database().execute(\n \"SELECT column_type, config FROM column_types \"\n \"WHERE database_name = ? AND resource_name = ? AND column_name = ?\",\n [database, resource, column],\n )\n rows = row.rows\n if not rows:\n return None\n ct_name, config = rows[0]\n ct_cls = self._column_types.get(ct_name)\n if ct_cls is None:\n return None\n column_detail = (\n await self._get_resource_column_details(database, resource)\n ).get(column)\n if not self._column_type_is_applicable(ct_cls, column_detail):\n return None\n return ct_cls(config=json.loads(config) if config else None)\n\n async def get_column_types(self, database: str, resource: str) -> dict:\n \"\"\"\n Return {column_name: ColumnType instance (with config)}\n for all columns with assigned types on the given resource.\n \"\"\"\n rows = await self.get_internal_database().execute(\n \"SELECT column_name, column_type, config FROM column_types \"\n \"WHERE database_name = ? AND resource_name = ?\",\n [database, resource],\n )\n column_details = await self._get_resource_column_details(database, resource)\n result = {}\n for row in rows.rows:\n col_name, ct_name, config = row\n ct_cls = self._column_types.get(ct_name)\n if ct_cls is not None and self._column_type_is_applicable(\n ct_cls, column_details.get(col_name)\n ):\n result[col_name] = ct_cls(config=json.loads(config) if config else None)\n return result\n\n async def set_column_type(\n self,\n database: str,\n resource: str,\n column: str,\n column_type: str,\n config: dict = None,\n ) -> None:\n \"\"\"Assign a column type. Overwrites any existing assignment.\"\"\"\n ct_cls = self._column_types.get(column_type)\n if ct_cls is not None:\n await self._validate_column_type_assignment(\n database, resource, column, ct_cls\n )\n await self.get_internal_database().execute_write(\n \"\"\"INSERT OR REPLACE INTO column_types\n (database_name, resource_name, column_name, column_type, config)\n VALUES (?, ?, ?, ?, ?)\"\"\",\n [\n database,\n resource,\n column,\n column_type,\n json.dumps(config) if config else None,\n ],\n )\n\n async def remove_column_type(\n self, database: str, resource: str, column: str\n ) -> None:\n \"\"\"Remove a column type assignment.\"\"\"\n await self.get_internal_database().execute_write(\n \"DELETE FROM column_types \"\n \"WHERE database_name = ? AND resource_name = ? AND column_name = ?\",\n [database, resource, column],\n )\n\n def get_internal_database(self):\n return self._internal_database\n\n def plugin_config(self, plugin_name, database=None, table=None, fallback=True):\n \"\"\"Return config for plugin, falling back from specified database/table\"\"\"\n if database is None and table is None:\n config = self._plugin_config_top(plugin_name)\n else:\n config = self._plugin_config_nested(plugin_name, database, table, fallback)\n\n return resolve_env_secrets(config, os.environ)\n\n def _plugin_config_top(self, plugin_name):\n \"\"\"Returns any top-level plugin configuration for the specified plugin.\"\"\"\n return ((self.config or {}).get(\"plugins\") or {}).get(plugin_name)\n\n def _plugin_config_nested(self, plugin_name, database, table=None, fallback=True):\n \"\"\"Returns any database or table-level plugin configuration for the specified plugin.\"\"\"\n db_config = ((self.config or {}).get(\"databases\") or {}).get(database)\n\n # if there's no db-level configuration, then return early, falling back to top-level if needed\n if not db_config:\n return self._plugin_config_top(plugin_name) if fallback else None\n\n db_plugin_config = (db_config.get(\"plugins\") or {}).get(plugin_name)\n\n if table:\n table_plugin_config = (\n ((db_config.get(\"tables\") or {}).get(table) or {}).get(\"plugins\") or {}\n ).get(plugin_name)\n\n # fallback to db_config or top-level config, in that order, if needed\n if table_plugin_config is None and fallback:\n return db_plugin_config or self._plugin_config_top(plugin_name)\n\n return table_plugin_config\n\n # fallback to top-level if needed\n if db_plugin_config is None and fallback:\n self._plugin_config_top(plugin_name)\n\n return db_plugin_config\n\n def app_css_hash(self):\n if not hasattr(self, \"_app_css_hash\"):\n with open(os.path.join(str(app_root), \"datasette/static/app.css\")) as fp:\n self._app_css_hash = hashlib.sha1(fp.read().encode(\"utf8\")).hexdigest()[\n :6\n ]\n return self._app_css_hash\n\n async def get_canned_queries(self, database_name, actor):\n queries = {}\n for more_queries in pm.hook.canned_queries(\n datasette=self,\n database=database_name,\n actor=actor,\n ):\n more_queries = await await_me_maybe(more_queries)\n queries.update(more_queries or {})\n # Fix any {\"name\": \"select ...\"} queries to be {\"name\": {\"sql\": \"select ...\"}}\n for key in queries:\n if not isinstance(queries[key], dict):\n queries[key] = {\"sql\": queries[key]}\n # Also make sure \"name\" is available:\n queries[key][\"name\"] = key\n return queries\n\n async def get_canned_query(self, database_name, query_name, actor):\n queries = await self.get_canned_queries(database_name, actor)\n query = queries.get(query_name)\n if query:\n return query\n\n def _prepare_connection(self, conn, database):\n conn.row_factory = sqlite3.Row\n conn.text_factory = lambda x: str(x, \"utf-8\", \"replace\")\n if self.sqlite_extensions and database != INTERNAL_DB_NAME:\n conn.enable_load_extension(True)\n for extension in self.sqlite_extensions:\n # \"extension\" is either a string path to the extension\n # or a 2-item tuple that specifies which entrypoint to load.\n if isinstance(extension, tuple):\n path, entrypoint = extension\n conn.execute(\"SELECT load_extension(?, ?)\", [path, entrypoint])\n else:\n conn.execute(\"SELECT load_extension(?)\", [extension])\n if self.setting(\"cache_size_kb\"):\n conn.execute(f\"PRAGMA cache_size=-{self.setting('cache_size_kb')}\")\n # pylint: disable=no-member\n if database != INTERNAL_DB_NAME:\n pm.hook.prepare_connection(conn=conn, database=database, datasette=self)\n # If self.crossdb and this is _memory, connect the first SQLITE_LIMIT_ATTACHED databases\n if self.crossdb and database == \"_memory\":\n count = 0\n for db_name, db in self.databases.items():\n if count >= SQLITE_LIMIT_ATTACHED or db.is_memory:\n continue\n sql = 'ATTACH DATABASE \"file:{path}?{qs}\" AS [{name}];'.format(\n path=db.path,\n qs=\"mode=ro\" if db.is_mutable else \"immutable=1\",\n name=db_name,\n )\n conn.execute(sql)\n count += 1\n\n def add_message(self, request, message, type=INFO):\n if not hasattr(request, \"_messages\"):\n request._messages = []\n request._messages_should_clear = False\n request._messages.append((message, type))\n\n def _write_messages_to_response(self, request, response):\n if getattr(request, \"_messages\", None):\n # Set those messages\n response.set_cookie(\"ds_messages\", self.sign(request._messages, \"messages\"))\n elif getattr(request, \"_messages_should_clear\", False):\n response.set_cookie(\"ds_messages\", \"\", expires=0, max_age=0)\n\n def _show_messages(self, request):\n if getattr(request, \"_messages\", None):\n request._messages_should_clear = True\n messages = request._messages\n request._messages = []\n return messages\n else:\n return []\n\n async def _crumb_items(self, request, table=None, database=None):\n crumbs = []\n actor = None\n if request:\n actor = request.actor\n # Top-level link\n if await self.allowed(action=\"view-instance\", actor=actor):\n crumbs.append({\"href\": self.urls.instance(), \"label\": \"home\"})\n # Database link\n if database:\n if await self.allowed(\n action=\"view-database\",\n resource=DatabaseResource(database=database),\n actor=actor,\n ):\n crumbs.append(\n {\n \"href\": self.urls.database(database),\n \"label\": database,\n }\n )\n # Table link\n if table:\n assert database, \"table= requires database=\"\n if await self.allowed(\n action=\"view-table\",\n resource=TableResource(database=database, table=table),\n actor=actor,\n ):\n crumbs.append(\n {\n \"href\": self.urls.table(database, table),\n \"label\": table,\n }\n )\n return crumbs\n\n async def actors_from_ids(\n self, actor_ids: Iterable[str | int]\n ) -> Dict[int | str, Dict]:\n result = pm.hook.actors_from_ids(datasette=self, actor_ids=actor_ids)\n if result is None:\n # Do the default thing\n return {actor_id: {\"id\": actor_id} for actor_id in actor_ids}\n result = await await_me_maybe(result)\n return result\n\n async def track_event(self, event: Event):\n assert isinstance(event, self.event_classes), \"Invalid event type: {}\".format(\n type(event)\n )\n for hook in pm.hook.track_event(datasette=self, event=event):\n await await_me_maybe(hook)\n\n def resource_for_action(self, action: str, parent: str | None, child: str | None):\n \"\"\"\n Create a Resource instance for the given action with parent/child values.\n\n Looks up the action's resource_class and instantiates it with the\n provided parent and child identifiers.\n\n Args:\n action: The action name (e.g., \"view-table\", \"view-query\")\n parent: The parent resource identifier (e.g., database name)\n child: The child resource identifier (e.g., table/query name)\n\n Returns:\n A Resource instance of the appropriate subclass\n\n Raises:\n ValueError: If the action is unknown\n \"\"\"\n from datasette.permissions import Resource\n\n action_obj = self.actions.get(action)\n if not action_obj:\n raise ValueError(f\"Unknown action: {action}\")\n\n resource_class = action_obj.resource_class\n instance = object.__new__(resource_class)\n Resource.__init__(instance, parent=parent, child=child)\n return instance\n\n async def check_visibility(\n self,\n actor: dict,\n action: str,\n resource: \"Resource\" | None = None,\n ):\n \"\"\"\n Check if actor can see a resource and if it's private.\n\n Returns (visible, private) tuple:\n - visible: bool - can the actor see it?\n - private: bool - if visible, can anonymous users NOT see it?\n \"\"\"\n from datasette.permissions import Resource\n\n # Validate that resource is a Resource object or None\n if resource is not None and not isinstance(resource, Resource):\n raise TypeError(\"resource must be a Resource subclass instance or None.\")\n\n # Check if actor can see it\n if not await self.allowed(action=action, resource=resource, actor=actor):\n return False, False\n\n # Check if anonymous user can see it (for \"private\" flag)\n if not await self.allowed(action=action, resource=resource, actor=None):\n # Actor can see it but anonymous cannot - it's private\n return True, True\n\n # Both actor and anonymous can see it - it's public\n return True, False\n\n async def allowed_resources_sql(\n self,\n *,\n action: str,\n actor: dict | None = None,\n parent: str | None = None,\n include_is_private: bool = False,\n ) -> ResourcesSQL:\n \"\"\"\n Build SQL query to get all resources the actor can access for the given action.\n\n Args:\n action: The action name (e.g., \"view-table\")\n actor: The actor dict (or None for unauthenticated)\n parent: Optional parent filter (e.g., database name) to limit results\n include_is_private: If True, include is_private column showing if anonymous cannot access\n\n Returns a namedtuple of (query: str, params: dict) that can be executed against the internal database.\n The query returns rows with (parent, child, reason) columns, plus is_private if requested.\n\n Example:\n query, params = await datasette.allowed_resources_sql(\n action=\"view-table\",\n actor=actor,\n parent=\"mydb\",\n include_is_private=True\n )\n result = await datasette.get_internal_database().execute(query, params)\n \"\"\"\n from datasette.utils.actions_sql import build_allowed_resources_sql\n\n action_obj = self.actions.get(action)\n if not action_obj:\n raise ValueError(f\"Unknown action: {action}\")\n\n sql, params = await build_allowed_resources_sql(\n self, actor, action, parent=parent, include_is_private=include_is_private\n )\n return ResourcesSQL(sql, params)\n\n async def allowed_resources(\n self,\n action: str,\n actor: dict | None = None,\n *,\n parent: str | None = None,\n include_is_private: bool = False,\n include_reasons: bool = False,\n limit: int = 100,\n next: str | None = None,\n ) -> PaginatedResources:\n \"\"\"\n Return paginated resources the actor can access for the given action.\n\n Uses SQL with keyset pagination to efficiently filter resources.\n Returns PaginatedResources with list of Resource instances and pagination metadata.\n\n Args:\n action: The action name (e.g., \"view-table\")\n actor: The actor dict (or None for unauthenticated)\n parent: Optional parent filter (e.g., database name) to limit results\n include_is_private: If True, adds a .private attribute to each Resource\n include_reasons: If True, adds a .reasons attribute with List[str] of permission reasons\n limit: Maximum number of results to return (1-1000, default 100)\n next: Keyset token from previous page for pagination\n\n Returns:\n PaginatedResources with:\n - resources: List of Resource objects for this page\n - next: Token for next page (None if no more results)\n\n Example:\n # Get first page of tables\n page = await datasette.allowed_resources(\"view-table\", actor, limit=50)\n for table in page.resources:\n print(f\"{table.parent}/{table.child}\")\n\n # Get next page\n if page.next:\n next_page = await datasette.allowed_resources(\n \"view-table\", actor, limit=50, next=page.next\n )\n\n # With reasons for debugging\n page = await datasette.allowed_resources(\n \"view-table\", actor, include_reasons=True\n )\n for table in page.resources:\n print(f\"{table.child}: {table.reasons}\")\n\n # Iterate through all results with async generator\n page = await datasette.allowed_resources(\"view-table\", actor)\n async for table in page.all():\n print(table.child)\n \"\"\"\n\n action_obj = self.actions.get(action)\n if not action_obj:\n raise ValueError(f\"Unknown action: {action}\")\n\n # Validate and cap limit\n limit = min(max(1, limit), 1000)\n\n # Get base SQL query\n query, params = await self.allowed_resources_sql(\n action=action,\n actor=actor,\n parent=parent,\n include_is_private=include_is_private,\n )\n\n # Add keyset pagination WHERE clause if next token provided\n if next:\n try:\n components = urlsafe_components(next)\n if len(components) >= 2:\n last_parent, last_child = components[0], components[1]\n # Keyset condition: (parent > last) OR (parent = last AND child > last)\n keyset_where = \"\"\"\n (parent > :keyset_parent OR\n (parent = :keyset_parent AND child > :keyset_child))\n \"\"\"\n # Wrap original query and add keyset filter\n query = f\"SELECT * FROM ({query}) WHERE {keyset_where}\"\n params[\"keyset_parent\"] = last_parent\n params[\"keyset_child\"] = last_child\n except (ValueError, KeyError):\n # Invalid token - ignore and start from beginning\n pass\n\n # Add LIMIT (fetch limit+1 to detect if there are more results)\n # Note: query from allowed_resources_sql() already includes ORDER BY parent, child\n query = f\"{query} LIMIT :limit\"\n params[\"limit\"] = limit + 1\n\n # Execute query\n result = await self.get_internal_database().execute(query, params)\n rows = list(result.rows)\n\n # Check if truncated (got more than limit rows)\n truncated = len(rows) > limit\n if truncated:\n rows = rows[:limit] # Remove the extra row\n\n # Build Resource objects with optional attributes\n resources = []\n for row in rows:\n # row[0]=parent, row[1]=child, row[2]=reason, row[3]=is_private (if requested)\n resource = self.resource_for_action(action, parent=row[0], child=row[1])\n\n # Add reasons if requested\n if include_reasons:\n reason_json = row[2]\n try:\n reasons_array = (\n json.loads(reason_json) if isinstance(reason_json, str) else []\n )\n resource.reasons = [r for r in reasons_array if r is not None]\n except (json.JSONDecodeError, TypeError):\n resource.reasons = [reason_json] if reason_json else []\n\n # Add private flag if requested\n if include_is_private:\n resource.private = bool(row[3])\n\n resources.append(resource)\n\n # Generate next token if there are more results\n next_token = None\n if truncated and resources:\n last_resource = resources[-1]\n # Use tilde-encoding like table pagination\n next_token = \"{},{}\".format(\n tilde_encode(str(last_resource.parent)),\n tilde_encode(str(last_resource.child)),\n )\n\n return PaginatedResources(\n resources=resources,\n next=next_token,\n _datasette=self,\n _action=action,\n _actor=actor,\n _parent=parent,\n _include_is_private=include_is_private,\n _include_reasons=include_reasons,\n _limit=limit,\n )\n\n async def allowed(\n self,\n *,\n action: str,\n resource: \"Resource\" = None,\n actor: dict | None = None,\n ) -> bool:\n \"\"\"\n Check if actor can perform action on specific resource.\n\n Uses SQL to check permission for a single resource without fetching all resources.\n This is efficient - it does NOT call allowed_resources() and check membership.\n\n For global actions, resource should be None (or omitted).\n\n Example:\n from datasette.resources import TableResource\n can_view = await datasette.allowed(\n action=\"view-table\",\n resource=TableResource(database=\"analytics\", table=\"users\"),\n actor=actor\n )\n\n # For global actions, resource can be omitted:\n can_debug = await datasette.allowed(action=\"permissions-debug\", actor=actor)\n \"\"\"\n from datasette.utils.actions_sql import check_permission_for_resource\n\n # For global actions, resource remains None\n\n # Check if this action has also_requires - if so, check that action first\n action_obj = self.actions.get(action)\n if action_obj and action_obj.also_requires:\n # Must have the required action first\n if not await self.allowed(\n action=action_obj.also_requires,\n resource=resource,\n actor=actor,\n ):\n return False\n\n # For global actions, resource is None\n parent = resource.parent if resource else None\n child = resource.child if resource else None\n\n result = await check_permission_for_resource(\n datasette=self,\n actor=actor,\n action=action,\n parent=parent,\n child=child,\n )\n\n # Log the permission check for debugging\n self._permission_checks.append(\n PermissionCheck(\n when=datetime.datetime.now(datetime.timezone.utc).isoformat(),\n actor=actor,\n action=action,\n parent=parent,\n child=child,\n result=result,\n )\n )\n\n return result\n\n async def ensure_permission(\n self,\n *,\n action: str,\n resource: \"Resource\" = None,\n actor: dict | None = None,\n ):\n \"\"\"\n Check if actor can perform action on resource, raising Forbidden if not.\n\n This is a convenience wrapper around allowed() that raises Forbidden\n instead of returning False. Use this when you want to enforce a permission\n check and halt execution if it fails.\n\n Example:\n from datasette.resources import TableResource\n\n # Will raise Forbidden if actor cannot view the table\n await datasette.ensure_permission(\n action=\"view-table\",\n resource=TableResource(database=\"analytics\", table=\"users\"),\n actor=request.actor\n )\n\n # For instance-level actions, resource can be omitted:\n await datasette.ensure_permission(\n action=\"permissions-debug\",\n actor=request.actor\n )\n \"\"\"\n if not await self.allowed(action=action, resource=resource, actor=actor):\n raise Forbidden(action)\n\n async def execute(\n self,\n db_name,\n sql,\n params=None,\n truncate=False,\n custom_time_limit=None,\n page_size=None,\n log_sql_errors=True,\n ):\n return await self.databases[db_name].execute(\n sql,\n params=params,\n truncate=truncate,\n custom_time_limit=custom_time_limit,\n page_size=page_size,\n log_sql_errors=log_sql_errors,\n )\n\n async def expand_foreign_keys(self, actor, database, table, column, values):\n \"\"\"Returns dict mapping (column, value) -> label\"\"\"\n labeled_fks = {}\n db = self.databases[database]\n foreign_keys = await db.foreign_keys_for_table(table)\n # Find the foreign_key for this column\n try:\n fk = [\n foreign_key\n for foreign_key in foreign_keys\n if foreign_key[\"column\"] == column\n ][0]\n except IndexError:\n return {}\n # Ensure user has permission to view the referenced table\n from datasette.resources import TableResource\n\n other_table = fk[\"other_table\"]\n other_column = fk[\"other_column\"]\n visible, _ = await self.check_visibility(\n actor,\n action=\"view-table\",\n resource=TableResource(database=database, table=other_table),\n )\n if not visible:\n return {}\n label_column = await db.label_column_for_table(other_table)\n if not label_column:\n return {(fk[\"column\"], value): str(value) for value in values}\n labeled_fks = {}\n sql = \"\"\"\n select {other_column}, {label_column}\n from {other_table}\n where {other_column} in ({placeholders})\n \"\"\".format(\n other_column=escape_sqlite(other_column),\n label_column=escape_sqlite(label_column),\n other_table=escape_sqlite(other_table),\n placeholders=\", \".join([\"?\"] * len(set(values))),\n )\n try:\n results = await self.execute(database, sql, list(set(values)))\n except QueryInterrupted:\n pass\n else:\n for id, value in results:\n labeled_fks[(fk[\"column\"], id)] = value\n return labeled_fks\n\n def absolute_url(self, request, path):\n url = urllib.parse.urljoin(request.url, path)\n if url.startswith(\"http://\") and self.setting(\"force_https_urls\"):\n url = \"https://\" + url[len(\"http://\") :]\n return url\n\n def _connected_databases(self):\n return [\n {\n \"name\": d.name,\n \"route\": d.route,\n \"path\": d.path,\n \"size\": d.size,\n \"is_mutable\": d.is_mutable,\n \"is_memory\": d.is_memory,\n \"hash\": d.hash,\n }\n for name, d in self.databases.items()\n ]\n\n def _versions(self):\n conn = sqlite3.connect(\":memory:\")\n self._prepare_connection(conn, \"_memory\")\n sqlite_version = conn.execute(\"select sqlite_version()\").fetchone()[0]\n sqlite_extensions = {\"json1\": detect_json1(conn)}\n for extension, testsql, hasversion in (\n (\"spatialite\", \"SELECT spatialite_version()\", True),\n ):\n try:\n result = conn.execute(testsql)\n if hasversion:\n sqlite_extensions[extension] = result.fetchone()[0]\n else:\n sqlite_extensions[extension] = None\n except Exception:\n pass\n # More details on SpatiaLite\n if \"spatialite\" in sqlite_extensions:\n spatialite_details = {}\n for fn in SPATIALITE_FUNCTIONS:\n try:\n result = conn.execute(\"select {}()\".format(fn))\n spatialite_details[fn] = result.fetchone()[0]\n except Exception as e:\n spatialite_details[fn] = {\"error\": str(e)}\n sqlite_extensions[\"spatialite\"] = spatialite_details\n\n # Figure out supported FTS versions\n fts_versions = []\n for fts in (\"FTS5\", \"FTS4\", \"FTS3\"):\n try:\n conn.execute(\n \"CREATE VIRTUAL TABLE v{fts} USING {fts} (data)\".format(fts=fts)\n )\n fts_versions.append(fts)\n except sqlite3.OperationalError:\n continue\n datasette_version = {\"version\": __version__}\n if self.version_note:\n datasette_version[\"note\"] = self.version_note\n\n try:\n # Optional import to avoid breaking Pyodide\n # https://github.com/simonw/datasette/issues/1733#issuecomment-1115268245\n import uvicorn\n\n uvicorn_version = uvicorn.__version__\n except ImportError:\n uvicorn_version = None\n info = {\n \"python\": {\n \"version\": \".\".join(map(str, sys.version_info[:3])),\n \"full\": sys.version,\n },\n \"datasette\": datasette_version,\n \"asgi\": \"3.0\",\n \"uvicorn\": uvicorn_version,\n \"sqlite\": {\n \"version\": sqlite_version,\n \"fts_versions\": fts_versions,\n \"extensions\": sqlite_extensions,\n \"compile_options\": [\n r[0] for r in conn.execute(\"pragma compile_options;\").fetchall()\n ],\n },\n }\n if using_pysqlite3:\n for package in (\"pysqlite3\", \"pysqlite3-binary\"):\n try:\n info[\"pysqlite3\"] = importlib.metadata.version(package)\n break\n except importlib.metadata.PackageNotFoundError:\n pass\n return info\n\n def _plugins(self, request=None, all=False):\n ps = list(get_plugins())\n should_show_all = False\n if request is not None:\n should_show_all = request.args.get(\"all\")\n else:\n should_show_all = all\n if not should_show_all:\n ps = [p for p in ps if p[\"name\"] not in DEFAULT_PLUGINS]\n ps.sort(key=lambda p: p[\"name\"])\n return [\n {\n \"name\": p[\"name\"],\n \"static\": p[\"static_path\"] is not None,\n \"templates\": p[\"templates_path\"] is not None,\n \"version\": p.get(\"version\"),\n \"hooks\": list(sorted(set(p[\"hooks\"]))),\n }\n for p in ps\n ]\n\n def _threads(self):\n if self.setting(\"num_sql_threads\") == 0:\n return {\"num_threads\": 0, \"threads\": []}\n threads = list(threading.enumerate())\n d = {\n \"num_threads\": len(threads),\n \"threads\": [\n {\"name\": t.name, \"ident\": t.ident, \"daemon\": t.daemon} for t in threads\n ],\n }\n tasks = asyncio.all_tasks()\n d.update(\n {\n \"num_tasks\": len(tasks),\n \"tasks\": [_cleaner_task_str(t) for t in tasks],\n }\n )\n return d\n\n def _actor(self, request):\n return {\"actor\": request.actor}\n\n def _actions(self):\n return [\n {\n \"name\": action.name,\n \"abbr\": action.abbr,\n \"description\": action.description,\n \"takes_parent\": action.takes_parent,\n \"takes_child\": action.takes_child,\n \"resource_class\": (\n action.resource_class.__name__ if action.resource_class else None\n ),\n \"also_requires\": action.also_requires,\n }\n for action in sorted(self.actions.values(), key=lambda a: a.name)\n ]\n\n async def table_config(self, database: str, table: str) -> dict:\n \"\"\"Return dictionary of configuration for specified table\"\"\"\n return (\n (self.config or {})\n .get(\"databases\", {})\n .get(database, {})\n .get(\"tables\", {})\n .get(table, {})\n )\n\n def _register_renderers(self):\n \"\"\"Register output renderers which output data in custom formats.\"\"\"\n # Built-in renderers\n self.renderers[\"json\"] = (json_renderer, lambda: True)\n\n # Hooks\n hook_renderers = []\n # pylint: disable=no-member\n for hook in pm.hook.register_output_renderer(datasette=self):\n if type(hook) is list:\n hook_renderers += hook\n else:\n hook_renderers.append(hook)\n\n for renderer in hook_renderers:\n self.renderers[renderer[\"extension\"]] = (\n # It used to be called \"callback\" - remove this in Datasette 1.0\n renderer.get(\"render\") or renderer[\"callback\"],\n renderer.get(\"can_render\") or (lambda: True),\n )\n\n async def render_template(\n self,\n templates: List[str] | str | Template,\n context: Dict[str, Any] | Context | None = None,\n request: Request | None = None,\n view_name: str | None = None,\n ):\n if not self._startup_invoked:\n raise Exception(\"render_template() called before await ds.invoke_startup()\")\n context = context or {}\n if isinstance(templates, Template):\n template = templates\n else:\n if isinstance(templates, str):\n templates = [templates]\n template = self.get_jinja_environment(request).select_template(templates)\n if dataclasses.is_dataclass(context):\n context = dataclasses.asdict(context)\n body_scripts = []\n # pylint: disable=no-member\n for extra_script in pm.hook.extra_body_script(\n template=template.name,\n database=context.get(\"database\"),\n table=context.get(\"table\"),\n columns=context.get(\"columns\"),\n view_name=view_name,\n request=request,\n datasette=self,\n ):\n extra_script = await await_me_maybe(extra_script)\n if isinstance(extra_script, dict):\n script = extra_script[\"script\"]\n module = bool(extra_script.get(\"module\"))\n else:\n script = extra_script\n module = False\n body_scripts.append({\"script\": Markup(script), \"module\": module})\n\n extra_template_vars = {}\n # pylint: disable=no-member\n for extra_vars in pm.hook.extra_template_vars(\n template=template.name,\n database=context.get(\"database\"),\n table=context.get(\"table\"),\n columns=context.get(\"columns\"),\n view_name=view_name,\n request=request,\n datasette=self,\n ):\n extra_vars = await await_me_maybe(extra_vars)\n assert isinstance(extra_vars, dict), \"extra_vars is of type {}\".format(\n type(extra_vars)\n )\n extra_template_vars.update(extra_vars)\n\n async def menu_links():\n links = []\n for hook in pm.hook.menu_links(\n datasette=self,\n actor=request.actor if request else None,\n request=request or None,\n ):\n extra_links = await await_me_maybe(hook)\n if extra_links:\n links.extend(extra_links)\n return links\n\n template_context = {\n **context,\n **{\n \"request\": request,\n \"crumb_items\": self._crumb_items,\n \"urls\": self.urls,\n \"actor\": request.actor if request else None,\n \"menu_links\": menu_links,\n \"display_actor\": display_actor,\n \"show_logout\": request is not None\n and \"ds_actor\" in request.cookies\n and request.actor,\n \"app_css_hash\": self.app_css_hash(),\n \"zip\": zip,\n \"body_scripts\": body_scripts,\n \"format_bytes\": format_bytes,\n \"show_messages\": lambda: self._show_messages(request),\n \"extra_css_urls\": await self._asset_urls(\n \"extra_css_urls\", template, context, request, view_name\n ),\n \"extra_js_urls\": await self._asset_urls(\n \"extra_js_urls\", template, context, request, view_name\n ),\n \"base_url\": self.setting(\"base_url\"),\n \"csrftoken\": request.scope[\"csrftoken\"] if request else lambda: \"\",\n \"datasette_version\": __version__,\n },\n **extra_template_vars,\n }\n if request and request.args.get(\"_context\") and self.setting(\"template_debug\"):\n return \"
{}
\".format(\n escape(json.dumps(template_context, default=repr, indent=4))\n )\n\n return await template.render_async(template_context)\n\n def set_actor_cookie(\n self, response: Response, actor: dict, expire_after: int | None = None\n ):\n data = {\"a\": actor}\n if expire_after:\n expires_at = int(time.time()) + (24 * 60 * 60)\n data[\"e\"] = baseconv.base62.encode(expires_at)\n response.set_cookie(\"ds_actor\", self.sign(data, \"actor\"))\n\n def delete_actor_cookie(self, response: Response):\n response.set_cookie(\"ds_actor\", \"\", expires=0, max_age=0)\n\n async def _asset_urls(self, key, template, context, request, view_name):\n # Flatten list-of-lists from plugins:\n seen_urls = set()\n collected = []\n for hook in getattr(pm.hook, key)(\n template=template.name,\n database=context.get(\"database\"),\n table=context.get(\"table\"),\n columns=context.get(\"columns\"),\n view_name=view_name,\n request=request,\n datasette=self,\n ):\n hook = await await_me_maybe(hook)\n collected.extend(hook)\n collected.extend((self.config or {}).get(key) or [])\n output = []\n for url_or_dict in collected:\n if isinstance(url_or_dict, dict):\n url = url_or_dict[\"url\"]\n sri = url_or_dict.get(\"sri\")\n module = bool(url_or_dict.get(\"module\"))\n else:\n url = url_or_dict\n sri = None\n module = False\n if url in seen_urls:\n continue\n seen_urls.add(url)\n if url.startswith(\"/\"):\n # Take base_url into account:\n url = self.urls.path(url)\n script = {\"url\": url}\n if sri:\n script[\"sri\"] = sri\n if module:\n script[\"module\"] = True\n output.append(script)\n return output\n\n def _config(self):\n return redact_keys(\n self.config, (\"secret\", \"key\", \"password\", \"token\", \"hash\", \"dsn\")\n )\n\n def _routes(self):\n routes = []\n\n for routes_to_add in pm.hook.register_routes(datasette=self):\n for regex, view_fn in routes_to_add:\n routes.append((regex, wrap_view(view_fn, self)))\n\n def add_route(view, regex):\n routes.append((regex, view))\n\n add_route(IndexView.as_view(self), r\"/(\\.(?Pjsono?))?$\")\n add_route(IndexView.as_view(self), r\"/-/(\\.(?Pjsono?))?$\")\n add_route(permanent_redirect(\"/-/\"), r\"/-$\")\n # TODO: /favicon.ico and /-/static/ deserve far-future cache expires\n add_route(favicon, \"/favicon.ico\")\n\n add_route(\n asgi_static(app_root / \"datasette\" / \"static\"), r\"/-/static/(?P.*)$\"\n )\n for path, dirname in self.static_mounts:\n add_route(asgi_static(dirname), r\"/\" + path + \"/(?P.*)$\")\n\n # Mount any plugin static/ directories\n for plugin in get_plugins():\n if plugin[\"static_path\"]:\n add_route(\n asgi_static(plugin[\"static_path\"]),\n f\"/-/static-plugins/{plugin['name']}/(?P.*)$\",\n )\n # Support underscores in name in addition to hyphens, see https://github.com/simonw/datasette/issues/611\n add_route(\n asgi_static(plugin[\"static_path\"]),\n \"/-/static-plugins/{}/(?P.*)$\".format(\n plugin[\"name\"].replace(\"-\", \"_\")\n ),\n )\n add_route(\n permanent_redirect(\n \"/_memory\", forward_query_string=True, forward_rest=True\n ),\n r\"/:memory:(?P.*)$\",\n )\n add_route(\n JsonDataView.as_view(self, \"versions.json\", self._versions),\n r\"/-/versions(\\.(?Pjson))?$\",\n )\n add_route(\n JsonDataView.as_view(\n self, \"plugins.json\", self._plugins, needs_request=True\n ),\n r\"/-/plugins(\\.(?Pjson))?$\",\n )\n add_route(\n JsonDataView.as_view(self, \"settings.json\", lambda: self._settings),\n r\"/-/settings(\\.(?Pjson))?$\",\n )\n add_route(\n JsonDataView.as_view(self, \"config.json\", lambda: self._config()),\n r\"/-/config(\\.(?Pjson))?$\",\n )\n add_route(\n JsonDataView.as_view(self, \"threads.json\", self._threads),\n r\"/-/threads(\\.(?Pjson))?$\",\n )\n add_route(\n JsonDataView.as_view(self, \"databases.json\", self._connected_databases),\n r\"/-/databases(\\.(?Pjson))?$\",\n )\n add_route(\n JsonDataView.as_view(\n self, \"actor.json\", self._actor, needs_request=True, permission=None\n ),\n r\"/-/actor(\\.(?Pjson))?$\",\n )\n add_route(\n JsonDataView.as_view(\n self,\n \"actions.json\",\n self._actions,\n template=\"debug_actions.html\",\n permission=\"permissions-debug\",\n ),\n r\"/-/actions(\\.(?Pjson))?$\",\n )\n add_route(\n AuthTokenView.as_view(self),\n r\"/-/auth-token$\",\n )\n add_route(\n CreateTokenView.as_view(self),\n r\"/-/create-token$\",\n )\n add_route(\n ApiExplorerView.as_view(self),\n r\"/-/api$\",\n )\n add_route(\n TablesView.as_view(self),\n r\"/-/tables(\\.(?Pjson))?$\",\n )\n add_route(\n InstanceSchemaView.as_view(self),\n r\"/-/schema(\\.(?Pjson|md))?$\",\n )\n add_route(\n LogoutView.as_view(self),\n r\"/-/logout$\",\n )\n add_route(\n PermissionsDebugView.as_view(self),\n r\"/-/permissions$\",\n )\n add_route(\n AllowedResourcesView.as_view(self),\n r\"/-/allowed(\\.(?Pjson))?$\",\n )\n add_route(\n PermissionRulesView.as_view(self),\n r\"/-/rules(\\.(?Pjson))?$\",\n )\n add_route(\n PermissionCheckView.as_view(self),\n r\"/-/check(\\.(?Pjson))?$\",\n )\n add_route(\n MessagesDebugView.as_view(self),\n r\"/-/messages$\",\n )\n add_route(\n AllowDebugView.as_view(self),\n r\"/-/allow-debug$\",\n )\n add_route(\n wrap_view(PatternPortfolioView, self),\n r\"/-/patterns$\",\n )\n add_route(\n wrap_view(database_download, self),\n r\"/(?P[^\\/\\.]+)\\.db$\",\n )\n add_route(\n wrap_view(DatabaseView, self),\n r\"/(?P[^\\/\\.]+)(\\.(?P\\w+))?$\",\n )\n add_route(TableCreateView.as_view(self), r\"/(?P[^\\/\\.]+)/-/create$\")\n add_route(\n DatabaseSchemaView.as_view(self),\n r\"/(?P[^\\/\\.]+)/-/schema(\\.(?Pjson|md))?$\",\n )\n add_route(\n wrap_view(QueryView, self),\n r\"/(?P[^\\/\\.]+)/-/query(\\.(?P\\w+))?$\",\n )\n add_route(\n wrap_view(table_view, self),\n r\"/(?P[^\\/\\.]+)/(?P[^\\/\\.]+)(\\.(?P\\w+))?$\",\n )\n add_route(\n RowView.as_view(self),\n r\"/(?P[^\\/\\.]+)/(?P
[^/]+?)/(?P[^/]+?)(\\.(?P\\w+))?$\",\n )\n add_route(\n TableInsertView.as_view(self),\n r\"/(?P[^\\/\\.]+)/(?P
[^\\/\\.]+)/-/insert$\",\n )\n add_route(\n TableUpsertView.as_view(self),\n r\"/(?P[^\\/\\.]+)/(?P
[^\\/\\.]+)/-/upsert$\",\n )\n add_route(\n TableSetColumnTypeView.as_view(self),\n r\"/(?P[^\\/\\.]+)/(?P
[^\\/\\.]+)/-/set-column-type$\",\n )\n add_route(\n TableDropView.as_view(self),\n r\"/(?P[^\\/\\.]+)/(?P
[^\\/\\.]+)/-/drop$\",\n )\n add_route(\n TableSchemaView.as_view(self),\n r\"/(?P[^\\/\\.]+)/(?P
[^\\/\\.]+)/-/schema(\\.(?Pjson|md))?$\",\n )\n add_route(\n RowDeleteView.as_view(self),\n r\"/(?P[^\\/\\.]+)/(?P
[^/]+?)/(?P[^/]+?)/-/delete$\",\n )\n add_route(\n RowUpdateView.as_view(self),\n r\"/(?P[^\\/\\.]+)/(?P
[^/]+?)/(?P[^/]+?)/-/update$\",\n )\n return [\n # Compile any strings to regular expressions\n ((re.compile(pattern) if isinstance(pattern, str) else pattern), view)\n for pattern, view in routes\n ]\n\n async def resolve_database(self, request):\n database_route = tilde_decode(request.url_vars[\"database\"])\n try:\n return self.get_database(route=database_route)\n except KeyError:\n raise DatabaseNotFound(database_route)\n\n async def resolve_table(self, request):\n db = await self.resolve_database(request)\n table_name = tilde_decode(request.url_vars[\"table\"])\n # Table must exist\n is_view = False\n table_exists = await db.table_exists(table_name)\n if not table_exists:\n is_view = await db.view_exists(table_name)\n if not (table_exists or is_view):\n raise TableNotFound(db.name, table_name)\n return ResolvedTable(db, table_name, is_view)\n\n async def resolve_row(self, request):\n db, table_name, _ = await self.resolve_table(request)\n pk_values = urlsafe_components(request.url_vars[\"pks\"])\n sql, params, pks = await row_sql_params_pks(db, table_name, pk_values)\n results = await db.execute(sql, params, truncate=True)\n row = results.first()\n if row is None:\n raise RowNotFound(db.name, table_name, pk_values)\n return ResolvedRow(db, table_name, sql, params, pks, pk_values, results.first())\n\n def app(self):\n \"\"\"Returns an ASGI app function that serves the whole of Datasette\"\"\"\n routes = self._routes()\n\n async def setup_db():\n # First time server starts up, calculate table counts for immutable databases\n for database in self.databases.values():\n if not database.is_mutable:\n await database.table_counts(limit=60 * 60 * 1000)\n\n async def custom_csrf_error(scope, send, message_id):\n await asgi_send(\n send,\n content=await self.render_template(\n \"csrf_error.html\",\n {\"message_id\": message_id, \"message_name\": Errors(message_id).name},\n ),\n status=403,\n content_type=\"text/html; charset=utf-8\",\n )\n\n asgi = asgi_csrf.asgi_csrf(\n DatasetteRouter(self, routes),\n signing_secret=self._secret,\n cookie_name=\"ds_csrftoken\",\n skip_if_scope=lambda scope: any(\n pm.hook.skip_csrf(datasette=self, scope=scope)\n ),\n send_csrf_failed=custom_csrf_error,\n )\n if self.setting(\"trace_debug\"):\n asgi = AsgiTracer(asgi)\n asgi = AsgiLifespan(asgi)\n asgi = AsgiRunOnFirstRequest(asgi, on_startup=[setup_db, self.invoke_startup])\n for wrapper in pm.hook.asgi_wrapper(datasette=self):\n asgi = wrapper(asgi)\n return asgi\n\n\nclass DatasetteRouter:\n def __init__(self, datasette, routes):\n self.ds = datasette\n self.routes = routes or []\n\n async def __call__(self, scope, receive, send):\n # Because we care about \"foo/bar\" v.s. \"foo%2Fbar\" we decode raw_path ourselves\n path = scope[\"path\"]\n raw_path = scope.get(\"raw_path\")\n if raw_path:\n path = raw_path.decode(\"ascii\")\n path = path.partition(\"?\")[0]\n return await self.route_path(scope, receive, send, path)\n\n async def route_path(self, scope, receive, send, path):\n # Strip off base_url if present before routing\n base_url = self.ds.setting(\"base_url\")\n if base_url != \"/\" and path.startswith(base_url):\n path = \"/\" + path[len(base_url) :]\n scope = dict(scope, route_path=path)\n request = Request(scope, receive)\n # Populate request_messages if ds_messages cookie is present\n try:\n request._messages = self.ds.unsign(\n request.cookies.get(\"ds_messages\", \"\"), \"messages\"\n )\n except BadSignature:\n pass\n\n scope_modifications = {}\n # Apply force_https_urls, if set\n if (\n self.ds.setting(\"force_https_urls\")\n and scope[\"type\"] == \"http\"\n and scope.get(\"scheme\") != \"https\"\n ):\n scope_modifications[\"scheme\"] = \"https\"\n # Handle authentication\n default_actor = scope.get(\"actor\") or None\n actor = None\n results = pm.hook.actor_from_request(datasette=self.ds, request=request)\n for result in results:\n result = await await_me_maybe(result)\n if result and actor is None:\n actor = result\n # Don't break — we must await all coroutines to avoid\n # \"coroutine was never awaited\" warnings\n scope_modifications[\"actor\"] = actor or default_actor\n scope = dict(scope, **scope_modifications)\n\n match, view = resolve_routes(self.routes, path)\n\n if match is None:\n return await self.handle_404(request, send)\n\n new_scope = dict(scope, url_route={\"kwargs\": match.groupdict()})\n request.scope = new_scope\n try:\n response = await view(request, send)\n if response:\n self.ds._write_messages_to_response(request, response)\n await response.asgi_send(send)\n return\n except NotFound as exception:\n return await self.handle_404(request, send, exception)\n except Forbidden as exception:\n # Try the forbidden() plugin hook\n for custom_response in pm.hook.forbidden(\n datasette=self.ds, request=request, message=exception.args[0]\n ):\n custom_response = await await_me_maybe(custom_response)\n assert (\n custom_response\n ), \"Default forbidden() hook should have been called\"\n return await custom_response.asgi_send(send)\n except Exception as exception:\n return await self.handle_exception(request, send, exception)\n\n async def handle_404(self, request, send, exception=None):\n # If path contains % encoding, redirect to tilde encoding\n if \"%\" in request.path:\n # Try the same path but with \"%\" replaced by \"~\"\n # and \"~\" replaced with \"~7E\"\n # and \".\" replaced with \"~2E\"\n new_path = (\n request.path.replace(\"~\", \"~7E\").replace(\"%\", \"~\").replace(\".\", \"~2E\")\n )\n if request.query_string:\n new_path += \"?{}\".format(request.query_string)\n await asgi_send_redirect(send, new_path)\n return\n # If URL has a trailing slash, redirect to URL without it\n path = request.scope.get(\n \"raw_path\", request.scope[\"path\"].encode(\"utf8\")\n ).partition(b\"?\")[0]\n context = {}\n if path.endswith(b\"/\"):\n path = path.rstrip(b\"/\")\n if request.scope[\"query_string\"]:\n path += b\"?\" + request.scope[\"query_string\"]\n await asgi_send_redirect(send, path.decode(\"latin1\"))\n else:\n # Is there a pages/* template matching this path?\n route_path = request.scope.get(\"route_path\", request.scope[\"path\"])\n # Jinja requires template names to use \"/\" even on Windows\n template_name = \"pages\" + route_path + \".html\"\n # Build a list of pages/blah/{name}.html matching expressions\n environment = self.ds.get_jinja_environment(request)\n pattern_templates = [\n filepath\n for filepath in environment.list_templates()\n if \"{\" in filepath and filepath.startswith(\"pages/\")\n ]\n page_routes = [\n (route_pattern_from_filepath(filepath[len(\"pages/\") :]), filepath)\n for filepath in pattern_templates\n ]\n try:\n template = environment.select_template([template_name])\n except TemplateNotFound:\n template = None\n if template is None:\n # Try for a pages/blah/{name}.html template match\n for regex, wildcard_template in page_routes:\n match = regex.match(route_path)\n if match is not None:\n context.update(match.groupdict())\n template = wildcard_template\n break\n\n if template:\n headers = {}\n status = [200]\n\n def custom_header(name, value):\n headers[name] = value\n return \"\"\n\n def custom_status(code):\n status[0] = code\n return \"\"\n\n def custom_redirect(location, code=302):\n status[0] = code\n headers[\"Location\"] = location\n return \"\"\n\n def raise_404(message=\"\"):\n raise NotFoundExplicit(message)\n\n context.update(\n {\n \"custom_header\": custom_header,\n \"custom_status\": custom_status,\n \"custom_redirect\": custom_redirect,\n \"raise_404\": raise_404,\n }\n )\n try:\n body = await self.ds.render_template(\n template,\n context,\n request=request,\n view_name=\"page\",\n )\n except NotFoundExplicit as e:\n await self.handle_exception(request, send, e)\n return\n # Pull content-type out into separate parameter\n content_type = \"text/html; charset=utf-8\"\n matches = [k for k in headers if k.lower() == \"content-type\"]\n if matches:\n content_type = headers[matches[0]]\n await asgi_send(\n send,\n body,\n status=status[0],\n headers=headers,\n content_type=content_type,\n )\n else:\n await self.handle_exception(request, send, exception or NotFound(\"404\"))\n\n async def handle_exception(self, request, send, exception):\n responses = []\n for hook in pm.hook.handle_exception(\n datasette=self.ds,\n request=request,\n exception=exception,\n ):\n response = await await_me_maybe(hook)\n if response is not None:\n responses.append(response)\n\n assert responses, \"Default exception handler should have returned something\"\n # Even if there are multiple responses use just the first one\n response = responses[0]\n await response.asgi_send(send)\n\n\n_cleaner_task_str_re = re.compile(r\"\\S*site-packages/\")\n\n\ndef _cleaner_task_str(task):\n s = str(task)\n # This has something like the following in it:\n # running at /Users/simonw/Dropbox/Development/datasette/venv-3.7.5/lib/python3.7/site-packages/uvicorn/main.py:361>\n # Clean up everything up to and including site-packages\n return _cleaner_task_str_re.sub(\"\", s)\n\n\ndef wrap_view(view_fn_or_class, datasette):\n is_function = isinstance(view_fn_or_class, types.FunctionType)\n if is_function:\n return wrap_view_function(view_fn_or_class, datasette)\n else:\n if not isinstance(view_fn_or_class, type):\n raise ValueError(\"view_fn_or_class must be a function or a class\")\n return wrap_view_class(view_fn_or_class, datasette)\n\n\ndef wrap_view_class(view_class, datasette):\n async def async_view_for_class(request, send):\n instance = view_class()\n if inspect.iscoroutinefunction(instance.__call__):\n return await async_call_with_supported_arguments(\n instance.__call__,\n scope=request.scope,\n receive=request.receive,\n send=send,\n request=request,\n datasette=datasette,\n )\n else:\n return call_with_supported_arguments(\n instance.__call__,\n scope=request.scope,\n receive=request.receive,\n send=send,\n request=request,\n datasette=datasette,\n )\n\n async_view_for_class.view_class = view_class\n return async_view_for_class\n\n\ndef wrap_view_function(view_fn, datasette):\n @functools.wraps(view_fn)\n async def async_view_fn(request, send):\n if inspect.iscoroutinefunction(view_fn):\n response = await async_call_with_supported_arguments(\n view_fn,\n scope=request.scope,\n receive=request.receive,\n send=send,\n request=request,\n datasette=datasette,\n )\n else:\n response = call_with_supported_arguments(\n view_fn,\n scope=request.scope,\n receive=request.receive,\n send=send,\n request=request,\n datasette=datasette,\n )\n if response is not None:\n return response\n\n return async_view_fn\n\n\ndef permanent_redirect(path, forward_query_string=False, forward_rest=False):\n return wrap_view(\n lambda request, send: Response.redirect(\n path\n + (request.url_vars[\"rest\"] if forward_rest else \"\")\n + (\n (\"?\" + request.query_string)\n if forward_query_string and request.query_string\n else \"\"\n ),\n status=301,\n ),\n datasette=None,\n )\n\n\n_curly_re = re.compile(r\"({.*?})\")\n\n\ndef route_pattern_from_filepath(filepath):\n # Drop the \".html\" suffix\n if filepath.endswith(\".html\"):\n filepath = filepath[: -len(\".html\")]\n re_bits = [\"/\"]\n for bit in _curly_re.split(filepath):\n if _curly_re.match(bit):\n re_bits.append(f\"(?P<{bit[1:-1]}>[^/]*)\")\n else:\n re_bits.append(re.escape(bit))\n return re.compile(\"^\" + \"\".join(re_bits) + \"$\")\n\n\nclass NotFoundExplicit(NotFound):\n pass\n\n\nclass DatasetteClient:\n \"\"\"Internal HTTP client for making requests to a Datasette instance.\n\n Used for testing and for internal operations that need to make HTTP requests\n to the Datasette app without going through an actual HTTP server.\n \"\"\"\n\n def __init__(self, ds):\n self.ds = ds\n\n @property\n def app(self):\n return self.ds.app()\n\n def actor_cookie(self, actor):\n # Utility method, mainly for tests\n return self.ds.sign({\"a\": actor}, \"actor\")\n\n def _fix(self, path, avoid_path_rewrites=False):\n if not isinstance(path, PrefixedUrlString) and not avoid_path_rewrites:\n path = self.ds.urls.path(path)\n if path.startswith(\"/\"):\n path = f\"http://localhost{path}\"\n return path\n\n async def _request(self, method, path, skip_permission_checks=False, **kwargs):\n from datasette.permissions import SkipPermissions\n\n with _DatasetteClientContext():\n if skip_permission_checks:\n with SkipPermissions():\n async with httpx.AsyncClient(\n transport=httpx.ASGITransport(app=self.app),\n cookies=kwargs.pop(\"cookies\", None),\n ) as client:\n return await getattr(client, method)(self._fix(path), **kwargs)\n else:\n async with httpx.AsyncClient(\n transport=httpx.ASGITransport(app=self.app),\n cookies=kwargs.pop(\"cookies\", None),\n ) as client:\n return await getattr(client, method)(self._fix(path), **kwargs)\n\n async def get(self, path, skip_permission_checks=False, **kwargs):\n return await self._request(\n \"get\", path, skip_permission_checks=skip_permission_checks, **kwargs\n )\n\n async def options(self, path, skip_permission_checks=False, **kwargs):\n return await self._request(\n \"options\", path, skip_permission_checks=skip_permission_checks, **kwargs\n )\n\n async def head(self, path, skip_permission_checks=False, **kwargs):\n return await self._request(\n \"head\", path, skip_permission_checks=skip_permission_checks, **kwargs\n )\n\n async def post(self, path, skip_permission_checks=False, **kwargs):\n return await self._request(\n \"post\", path, skip_permission_checks=skip_permission_checks, **kwargs\n )\n\n async def put(self, path, skip_permission_checks=False, **kwargs):\n return await self._request(\n \"put\", path, skip_permission_checks=skip_permission_checks, **kwargs\n )\n\n async def patch(self, path, skip_permission_checks=False, **kwargs):\n return await self._request(\n \"patch\", path, skip_permission_checks=skip_permission_checks, **kwargs\n )\n\n async def delete(self, path, skip_permission_checks=False, **kwargs):\n return await self._request(\n \"delete\", path, skip_permission_checks=skip_permission_checks, **kwargs\n )\n\n async def request(self, method, path, skip_permission_checks=False, **kwargs):\n \"\"\"Make an HTTP request with the specified method.\n\n Args:\n method: HTTP method (e.g., \"GET\", \"POST\", \"PUT\")\n path: The path to request\n skip_permission_checks: If True, bypass all permission checks for this request\n **kwargs: Additional arguments to pass to httpx\n\n Returns:\n httpx.Response: The response from the request\n \"\"\"\n from datasette.permissions import SkipPermissions\n\n avoid_path_rewrites = kwargs.pop(\"avoid_path_rewrites\", None)\n with _DatasetteClientContext():\n if skip_permission_checks:\n with SkipPermissions():\n async with httpx.AsyncClient(\n transport=httpx.ASGITransport(app=self.app),\n cookies=kwargs.pop(\"cookies\", None),\n ) as client:\n return await client.request(\n method, self._fix(path, avoid_path_rewrites), **kwargs\n )\n else:\n async with httpx.AsyncClient(\n transport=httpx.ASGITransport(app=self.app),\n cookies=kwargs.pop(\"cookies\", None),\n ) as client:\n return await client.request(\n method, self._fix(path, avoid_path_rewrites), **kwargs\n )\n" }, { "path": "datasette/blob_renderer.py", "content": "from datasette import hookimpl\nfrom datasette.utils.asgi import Response, BadRequest\nfrom datasette.utils import to_css_class\nimport hashlib\n\n_BLOB_COLUMN = \"_blob_column\"\n_BLOB_HASH = \"_blob_hash\"\n\n\nasync def render_blob(datasette, database, rows, columns, request, table, view_name):\n if _BLOB_COLUMN not in request.args:\n raise BadRequest(f\"?{_BLOB_COLUMN}= is required\")\n blob_column = request.args[_BLOB_COLUMN]\n if blob_column not in columns:\n raise BadRequest(f\"{blob_column} is not a valid column\")\n\n # If ?_blob_hash= provided, use that to select the row - otherwise use first row\n blob_hash = None\n if _BLOB_HASH in request.args:\n blob_hash = request.args[_BLOB_HASH]\n for row in rows:\n value = row[blob_column]\n if hashlib.sha256(value).hexdigest() == blob_hash:\n break\n else:\n # Loop did not break\n raise BadRequest(\n \"Link has expired - the requested binary content has changed or could not be found.\"\n )\n else:\n row = rows[0]\n\n value = row[blob_column]\n filename_bits = []\n if table:\n filename_bits.append(to_css_class(table))\n if \"pks\" in request.url_vars:\n filename_bits.append(request.url_vars[\"pks\"])\n filename_bits.append(to_css_class(blob_column))\n if blob_hash:\n filename_bits.append(blob_hash[:6])\n filename = \"-\".join(filename_bits) + \".blob\"\n headers = {\n \"X-Content-Type-Options\": \"nosniff\",\n \"Content-Disposition\": f'attachment; filename=\"{filename}\"',\n }\n return Response(\n body=value or b\"\",\n status=200,\n headers=headers,\n content_type=\"application/binary\",\n )\n\n\n@hookimpl\ndef register_output_renderer():\n return {\n \"extension\": \"blob\",\n \"render\": render_blob,\n \"can_render\": lambda: False,\n }\n" }, { "path": "datasette/cli.py", "content": "import asyncio\nimport uvicorn\nimport click\nfrom click import formatting\nfrom click.types import CompositeParamType\nfrom click_default_group import DefaultGroup\nimport functools\nimport json\nimport os\nimport pathlib\nfrom runpy import run_module\nimport shutil\nfrom subprocess import call\nimport sys\nimport textwrap\nimport webbrowser\nfrom .app import (\n Datasette,\n DEFAULT_SETTINGS,\n SETTINGS,\n SQLITE_LIMIT_ATTACHED,\n pm,\n)\nfrom .utils import (\n LoadExtension,\n StartupError,\n check_connection,\n deep_dict_update,\n find_spatialite,\n parse_metadata,\n ConnectionProblem,\n SpatialiteConnectionProblem,\n initial_path_for_datasette,\n pairs_to_nested_config,\n temporary_docker_directory,\n value_as_boolean,\n SpatialiteNotFound,\n StaticMount,\n ValueAsBooleanError,\n)\nfrom .utils.sqlite import sqlite3\nfrom .utils.testing import TestClient\nfrom .version import __version__\n\n\ndef run_sync(coro_func):\n \"\"\"Run an async callable to completion on a fresh event loop.\"\"\"\n loop = asyncio.new_event_loop()\n try:\n asyncio.set_event_loop(loop)\n return loop.run_until_complete(coro_func())\n finally:\n asyncio.set_event_loop(None)\n loop.close()\n\n\n# Use Rich for tracebacks if it is installed\ntry:\n from rich.traceback import install\n\n install(show_locals=True)\nexcept ImportError:\n pass\n\n\nclass Setting(CompositeParamType):\n name = \"setting\"\n arity = 2\n\n def convert(self, config, param, ctx):\n name, value = config\n if name in DEFAULT_SETTINGS:\n # For backwards compatibility with how this worked prior to\n # Datasette 1.0, we turn bare setting names into setting.name\n # Type checking for those older settings\n default = DEFAULT_SETTINGS[name]\n name = \"settings.{}\".format(name)\n if isinstance(default, bool):\n try:\n return name, \"true\" if value_as_boolean(value) else \"false\"\n except ValueAsBooleanError:\n self.fail(f'\"{name}\" should be on/off/true/false/1/0', param, ctx)\n elif isinstance(default, int):\n if not value.isdigit():\n self.fail(f'\"{name}\" should be an integer', param, ctx)\n return name, value\n elif isinstance(default, str):\n return name, value\n else:\n # Should never happen:\n self.fail(\"Invalid option\")\n return name, value\n\n\ndef sqlite_extensions(fn):\n fn = click.option(\n \"sqlite_extensions\",\n \"--load-extension\",\n type=LoadExtension(),\n envvar=\"DATASETTE_LOAD_EXTENSION\",\n multiple=True,\n help=\"Path to a SQLite extension to load, and optional entrypoint\",\n )(fn)\n\n # Wrap it in a custom error handler\n @functools.wraps(fn)\n def wrapped(*args, **kwargs):\n try:\n return fn(*args, **kwargs)\n except AttributeError as e:\n if \"enable_load_extension\" in str(e):\n raise click.ClickException(textwrap.dedent(\"\"\"\n Your Python installation does not have the ability to load SQLite extensions.\n\n More information: https://datasette.io/help/extensions\n \"\"\").strip())\n raise\n\n return wrapped\n\n\n@click.group(cls=DefaultGroup, default=\"serve\", default_if_no_args=True)\n@click.version_option(version=__version__)\ndef cli():\n \"\"\"\n Datasette is an open source multi-tool for exploring and publishing data\n\n \\b\n About Datasette: https://datasette.io/\n Full documentation: https://docs.datasette.io/\n \"\"\"\n\n\n@cli.command()\n@click.argument(\"files\", type=click.Path(exists=True), nargs=-1)\n@click.option(\"--inspect-file\", default=\"-\")\n@sqlite_extensions\ndef inspect(files, inspect_file, sqlite_extensions):\n \"\"\"\n Generate JSON summary of provided database files\n\n This can then be passed to \"datasette --inspect-file\" to speed up count\n operations against immutable database files.\n \"\"\"\n inspect_data = run_sync(lambda: inspect_(files, sqlite_extensions))\n if inspect_file == \"-\":\n sys.stdout.write(json.dumps(inspect_data, indent=2))\n else:\n with open(inspect_file, \"w\") as fp:\n fp.write(json.dumps(inspect_data, indent=2))\n\n\nasync def inspect_(files, sqlite_extensions):\n app = Datasette([], immutables=files, sqlite_extensions=sqlite_extensions)\n data = {}\n for name, database in app.databases.items():\n counts = await database.table_counts(limit=3600 * 1000)\n data[name] = {\n \"hash\": database.hash,\n \"size\": database.size,\n \"file\": database.path,\n \"tables\": {\n table_name: {\"count\": table_count}\n for table_name, table_count in counts.items()\n },\n }\n return data\n\n\n@cli.group()\ndef publish():\n \"\"\"Publish specified SQLite database files to the internet along with a Datasette-powered interface and API\"\"\"\n pass\n\n\n# Register publish plugins\npm.hook.publish_subcommand(publish=publish)\n\n\n@cli.command()\n@click.option(\"--all\", help=\"Include built-in default plugins\", is_flag=True)\n@click.option(\n \"--requirements\", help=\"Output requirements.txt of installed plugins\", is_flag=True\n)\n@click.option(\n \"--plugins-dir\",\n type=click.Path(exists=True, file_okay=False, dir_okay=True),\n help=\"Path to directory containing custom plugins\",\n)\ndef plugins(all, requirements, plugins_dir):\n \"\"\"List currently installed plugins\"\"\"\n app = Datasette([], plugins_dir=plugins_dir)\n if requirements:\n for plugin in app._plugins():\n if plugin[\"version\"]:\n click.echo(\"{}=={}\".format(plugin[\"name\"], plugin[\"version\"]))\n else:\n click.echo(json.dumps(app._plugins(all=all), indent=4))\n\n\n@cli.command()\n@click.argument(\"files\", type=click.Path(exists=True), nargs=-1, required=True)\n@click.option(\n \"-t\",\n \"--tag\",\n help=\"Name for the resulting Docker container, can optionally use name:tag format\",\n)\n@click.option(\n \"-m\",\n \"--metadata\",\n type=click.File(mode=\"r\"),\n help=\"Path to JSON/YAML file containing metadata to publish\",\n)\n@click.option(\"--extra-options\", help=\"Extra options to pass to datasette serve\")\n@click.option(\"--branch\", help=\"Install datasette from a GitHub branch e.g. main\")\n@click.option(\n \"--template-dir\",\n type=click.Path(exists=True, file_okay=False, dir_okay=True),\n help=\"Path to directory containing custom templates\",\n)\n@click.option(\n \"--plugins-dir\",\n type=click.Path(exists=True, file_okay=False, dir_okay=True),\n help=\"Path to directory containing custom plugins\",\n)\n@click.option(\n \"--static\",\n type=StaticMount(),\n help=\"Serve static files from this directory at /MOUNT/...\",\n multiple=True,\n)\n@click.option(\n \"--install\", help=\"Additional packages (e.g. plugins) to install\", multiple=True\n)\n@click.option(\"--spatialite\", is_flag=True, help=\"Enable SpatialLite extension\")\n@click.option(\"--version-note\", help=\"Additional note to show on /-/versions\")\n@click.option(\n \"--secret\",\n help=\"Secret used for signing secure values, such as signed cookies\",\n envvar=\"DATASETTE_PUBLISH_SECRET\",\n default=lambda: os.urandom(32).hex(),\n)\n@click.option(\n \"-p\",\n \"--port\",\n default=8001,\n type=click.IntRange(1, 65535),\n help=\"Port to run the server on, defaults to 8001\",\n)\n@click.option(\"--title\", help=\"Title for metadata\")\n@click.option(\"--license\", help=\"License label for metadata\")\n@click.option(\"--license_url\", help=\"License URL for metadata\")\n@click.option(\"--source\", help=\"Source label for metadata\")\n@click.option(\"--source_url\", help=\"Source URL for metadata\")\n@click.option(\"--about\", help=\"About label for metadata\")\n@click.option(\"--about_url\", help=\"About URL for metadata\")\ndef package(\n files,\n tag,\n metadata,\n extra_options,\n branch,\n template_dir,\n plugins_dir,\n static,\n install,\n spatialite,\n version_note,\n secret,\n port,\n **extra_metadata,\n):\n \"\"\"Package SQLite files into a Datasette Docker container\"\"\"\n if not shutil.which(\"docker\"):\n click.secho(\n ' The package command requires \"docker\" to be installed and configured ',\n bg=\"red\",\n fg=\"white\",\n bold=True,\n err=True,\n )\n sys.exit(1)\n with temporary_docker_directory(\n files,\n \"datasette\",\n metadata=metadata,\n extra_options=extra_options,\n branch=branch,\n template_dir=template_dir,\n plugins_dir=plugins_dir,\n static=static,\n install=install,\n spatialite=spatialite,\n version_note=version_note,\n secret=secret,\n extra_metadata=extra_metadata,\n port=port,\n ):\n args = [\"docker\", \"build\"]\n if tag:\n args.append(\"-t\")\n args.append(tag)\n args.append(\".\")\n call(args)\n\n\n@cli.command()\n@click.argument(\"packages\", nargs=-1)\n@click.option(\n \"-U\", \"--upgrade\", is_flag=True, help=\"Upgrade packages to latest version\"\n)\n@click.option(\n \"-r\",\n \"--requirement\",\n type=click.Path(exists=True),\n help=\"Install from requirements file\",\n)\n@click.option(\n \"-e\",\n \"--editable\",\n help=\"Install a project in editable mode from this path\",\n)\ndef install(packages, upgrade, requirement, editable):\n \"\"\"Install plugins and packages from PyPI into the same environment as Datasette\"\"\"\n if not packages and not requirement and not editable:\n raise click.UsageError(\"Please specify at least one package to install\")\n args = [\"pip\", \"install\"]\n if upgrade:\n args += [\"--upgrade\"]\n if editable:\n args += [\"--editable\", editable]\n if requirement:\n args += [\"-r\", requirement]\n args += list(packages)\n sys.argv = args\n run_module(\"pip\", run_name=\"__main__\")\n\n\n@cli.command()\n@click.argument(\"packages\", nargs=-1, required=True)\n@click.option(\"-y\", \"--yes\", is_flag=True, help=\"Don't ask for confirmation\")\ndef uninstall(packages, yes):\n \"\"\"Uninstall plugins and Python packages from the Datasette environment\"\"\"\n sys.argv = [\"pip\", \"uninstall\"] + list(packages) + ([\"-y\"] if yes else [])\n run_module(\"pip\", run_name=\"__main__\")\n\n\n@cli.command()\n@click.argument(\"files\", type=click.Path(), nargs=-1)\n@click.option(\n \"-i\",\n \"--immutable\",\n type=click.Path(exists=True),\n help=\"Database files to open in immutable mode\",\n multiple=True,\n)\n@click.option(\n \"-h\",\n \"--host\",\n default=\"127.0.0.1\",\n help=(\n \"Host for server. Defaults to 127.0.0.1 which means only connections \"\n \"from the local machine will be allowed. Use 0.0.0.0 to listen to \"\n \"all IPs and allow access from other machines.\"\n ),\n)\n@click.option(\n \"-p\",\n \"--port\",\n default=8001,\n type=click.IntRange(0, 65535),\n help=\"Port for server, defaults to 8001. Use -p 0 to automatically assign an available port.\",\n)\n@click.option(\n \"--uds\",\n help=\"Bind to a Unix domain socket\",\n)\n@click.option(\n \"--reload\",\n is_flag=True,\n help=\"Automatically reload if code or metadata change detected - useful for development\",\n)\n@click.option(\n \"--cors\", is_flag=True, help=\"Enable CORS by serving Access-Control-Allow-Origin: *\"\n)\n@sqlite_extensions\n@click.option(\n \"--inspect-file\", help='Path to JSON file created using \"datasette inspect\"'\n)\n@click.option(\n \"-m\",\n \"--metadata\",\n type=click.File(mode=\"r\"),\n help=\"Path to JSON/YAML file containing license/source metadata\",\n)\n@click.option(\n \"--template-dir\",\n type=click.Path(exists=True, file_okay=False, dir_okay=True),\n help=\"Path to directory containing custom templates\",\n)\n@click.option(\n \"--plugins-dir\",\n type=click.Path(exists=True, file_okay=False, dir_okay=True),\n help=\"Path to directory containing custom plugins\",\n)\n@click.option(\n \"--static\",\n type=StaticMount(),\n help=\"Serve static files from this directory at /MOUNT/...\",\n multiple=True,\n)\n@click.option(\"--memory\", is_flag=True, help=\"Make /_memory database available\")\n@click.option(\n \"-c\",\n \"--config\",\n type=click.File(mode=\"r\"),\n help=\"Path to JSON/YAML Datasette configuration file\",\n)\n@click.option(\n \"-s\",\n \"--setting\",\n \"settings\",\n type=Setting(),\n help=\"nested.key, value setting to use in Datasette configuration\",\n multiple=True,\n)\n@click.option(\n \"--secret\",\n help=\"Secret used for signing secure values, such as signed cookies\",\n envvar=\"DATASETTE_SECRET\",\n)\n@click.option(\n \"--root\",\n help=\"Output URL that sets a cookie authenticating the root user\",\n is_flag=True,\n)\n@click.option(\n \"--default-deny\",\n help=\"Deny all permissions by default\",\n is_flag=True,\n)\n@click.option(\n \"--get\",\n help=\"Run an HTTP GET request against this path, print results and exit\",\n)\n@click.option(\n \"--headers\",\n is_flag=True,\n help=\"Include HTTP headers in --get output\",\n)\n@click.option(\n \"--token\",\n help=\"API token to send with --get requests\",\n)\n@click.option(\n \"--actor\",\n help=\"Actor to use for --get requests (JSON string)\",\n)\n@click.option(\"--version-note\", help=\"Additional note to show on /-/versions\")\n@click.option(\"--help-settings\", is_flag=True, help=\"Show available settings\")\n@click.option(\"--pdb\", is_flag=True, help=\"Launch debugger on any errors\")\n@click.option(\n \"-o\",\n \"--open\",\n \"open_browser\",\n is_flag=True,\n help=\"Open Datasette in your web browser\",\n)\n@click.option(\n \"--create\",\n is_flag=True,\n help=\"Create database files if they do not exist\",\n)\n@click.option(\n \"--crossdb\",\n is_flag=True,\n help=\"Enable cross-database joins using the /_memory database\",\n)\n@click.option(\n \"--nolock\",\n is_flag=True,\n help=\"Ignore locking, open locked files in read-only mode\",\n)\n@click.option(\n \"--ssl-keyfile\",\n help=\"SSL key file\",\n envvar=\"DATASETTE_SSL_KEYFILE\",\n)\n@click.option(\n \"--ssl-certfile\",\n help=\"SSL certificate file\",\n envvar=\"DATASETTE_SSL_CERTFILE\",\n)\n@click.option(\n \"--internal\",\n type=click.Path(),\n help=\"Path to a persistent Datasette internal SQLite database\",\n)\ndef serve(\n files,\n immutable,\n host,\n port,\n uds,\n reload,\n cors,\n sqlite_extensions,\n inspect_file,\n metadata,\n template_dir,\n plugins_dir,\n static,\n memory,\n config,\n settings,\n secret,\n root,\n default_deny,\n get,\n headers,\n token,\n actor,\n version_note,\n help_settings,\n pdb,\n open_browser,\n create,\n crossdb,\n nolock,\n ssl_keyfile,\n ssl_certfile,\n internal,\n return_instance=False,\n):\n \"\"\"Serve up specified SQLite database files with a web UI\"\"\"\n if help_settings:\n formatter = formatting.HelpFormatter()\n with formatter.section(\"Settings\"):\n formatter.write_dl(\n [\n (option.name, f\"{option.help} (default={option.default})\")\n for option in SETTINGS\n ]\n )\n click.echo(formatter.getvalue())\n sys.exit(0)\n if reload:\n import hupper\n\n reloader = hupper.start_reloader(\"datasette.cli.cli\")\n if immutable:\n reloader.watch_files(immutable)\n if config:\n reloader.watch_files([config.name])\n if metadata:\n reloader.watch_files([metadata.name])\n\n inspect_data = None\n if inspect_file:\n with open(inspect_file) as fp:\n inspect_data = json.load(fp)\n\n metadata_data = None\n if metadata:\n metadata_data = parse_metadata(metadata.read())\n\n config_data = None\n if config:\n config_data = parse_metadata(config.read())\n\n config_data = config_data or {}\n\n # Merge in settings from -s/--setting\n if settings:\n settings_updates = pairs_to_nested_config(settings)\n # Merge recursively, to avoid over-writing nested values\n # https://github.com/simonw/datasette/issues/2389\n deep_dict_update(config_data, settings_updates)\n\n kwargs = dict(\n immutables=immutable,\n cache_headers=not reload,\n cors=cors,\n inspect_data=inspect_data,\n config=config_data,\n metadata=metadata_data,\n sqlite_extensions=sqlite_extensions,\n template_dir=template_dir,\n plugins_dir=plugins_dir,\n static_mounts=static,\n settings=None, # These are passed in config= now\n memory=memory,\n secret=secret,\n version_note=version_note,\n pdb=pdb,\n crossdb=crossdb,\n nolock=nolock,\n internal=internal,\n default_deny=default_deny,\n )\n\n # Separate directories from files\n directories = [f for f in files if os.path.isdir(f)]\n file_paths = [f for f in files if not os.path.isdir(f)]\n\n # Handle config_dir - only one directory allowed\n if len(directories) > 1:\n raise click.ClickException(\n \"Cannot pass multiple directories. Pass a single directory as config_dir.\"\n )\n elif len(directories) == 1:\n kwargs[\"config_dir\"] = pathlib.Path(directories[0])\n\n # Verify list of files, create if needed (and --create)\n for file in file_paths:\n if not pathlib.Path(file).exists():\n if create:\n sqlite3.connect(file).execute(\"vacuum\")\n else:\n raise click.ClickException(\n \"Invalid value for '[FILES]...': Path '{}' does not exist.\".format(\n file\n )\n )\n\n # Check for duplicate files by resolving all paths to their absolute forms\n # Collect all database files that will be loaded (explicit files + config_dir files)\n all_db_files = []\n\n # Add explicit files\n for file in file_paths:\n all_db_files.append((file, pathlib.Path(file).resolve()))\n\n # Add config_dir databases if config_dir is set\n if \"config_dir\" in kwargs:\n config_dir = kwargs[\"config_dir\"]\n for ext in (\"db\", \"sqlite\", \"sqlite3\"):\n for db_file in config_dir.glob(f\"*.{ext}\"):\n all_db_files.append((str(db_file), db_file.resolve()))\n\n # Check for duplicates\n seen = {}\n for original_path, resolved_path in all_db_files:\n if resolved_path in seen:\n raise click.ClickException(\n f\"Duplicate database file: '{original_path}' and '{seen[resolved_path]}' \"\n f\"both refer to {resolved_path}\"\n )\n seen[resolved_path] = original_path\n\n files = file_paths\n\n try:\n ds = Datasette(files, **kwargs)\n except SpatialiteNotFound:\n raise click.ClickException(\"Could not find SpatiaLite extension\")\n except StartupError as e:\n raise click.ClickException(e.args[0])\n\n if return_instance:\n # Private utility mechanism for writing unit tests\n return ds\n\n # Run async soundness checks before startup hooks, since invoke_startup\n # now populates internal tables which requires querying each database\n run_sync(lambda: check_databases(ds))\n\n # Run the \"startup\" plugin hooks\n try:\n run_sync(ds.invoke_startup)\n except StartupError as e:\n raise click.ClickException(e.args[0])\n\n if headers and not get:\n raise click.ClickException(\"--headers can only be used with --get\")\n\n if token and not get:\n raise click.ClickException(\"--token can only be used with --get\")\n\n if get:\n client = TestClient(ds)\n request_headers = {}\n if token:\n request_headers[\"Authorization\"] = \"Bearer {}\".format(token)\n cookies = {}\n if actor:\n cookies[\"ds_actor\"] = client.actor_cookie(json.loads(actor))\n response = client.get(get, headers=request_headers, cookies=cookies)\n\n if headers:\n # Output HTTP status code, headers, two newlines, then the response body\n click.echo(f\"HTTP/1.1 {response.status}\")\n for key, value in response.headers.items():\n click.echo(f\"{key}: {value}\")\n if response.text:\n click.echo()\n click.echo(response.text)\n else:\n click.echo(response.text)\n\n exit_code = 0 if response.status == 200 else 1\n sys.exit(exit_code)\n return\n\n # Start the server\n url = None\n if root:\n ds.root_enabled = True\n url = \"http://{}:{}{}?token={}\".format(\n host, port, ds.urls.path(\"-/auth-token\"), ds._root_token\n )\n click.echo(url)\n if open_browser:\n if url is None:\n # Figure out most convenient URL - to table, database or homepage\n path = run_sync(lambda: initial_path_for_datasette(ds))\n url = f\"http://{host}:{port}{path}\"\n webbrowser.open(url)\n uvicorn_kwargs = dict(\n host=host, port=port, log_level=\"info\", lifespan=\"on\", workers=1\n )\n if uds:\n uvicorn_kwargs[\"uds\"] = uds\n if ssl_keyfile:\n uvicorn_kwargs[\"ssl_keyfile\"] = ssl_keyfile\n if ssl_certfile:\n uvicorn_kwargs[\"ssl_certfile\"] = ssl_certfile\n uvicorn.run(ds.app(), **uvicorn_kwargs)\n\n\n@cli.command()\n@click.argument(\"id\")\n@click.option(\n \"--secret\",\n help=\"Secret used for signing the API tokens\",\n envvar=\"DATASETTE_SECRET\",\n required=True,\n)\n@click.option(\n \"-e\",\n \"--expires-after\",\n help=\"Token should expire after this many seconds\",\n type=int,\n)\n@click.option(\n \"alls\",\n \"-a\",\n \"--all\",\n type=str,\n metavar=\"ACTION\",\n multiple=True,\n help=\"Restrict token to this action\",\n)\n@click.option(\n \"databases\",\n \"-d\",\n \"--database\",\n type=(str, str),\n metavar=\"DB ACTION\",\n multiple=True,\n help=\"Restrict token to this action on this database\",\n)\n@click.option(\n \"resources\",\n \"-r\",\n \"--resource\",\n type=(str, str, str),\n metavar=\"DB RESOURCE ACTION\",\n multiple=True,\n help=\"Restrict token to this action on this database resource (a table, SQL view or named query)\",\n)\n@click.option(\n \"--debug\",\n help=\"Show decoded token\",\n is_flag=True,\n)\n@click.option(\n \"--plugins-dir\",\n type=click.Path(exists=True, file_okay=False, dir_okay=True),\n help=\"Path to directory containing custom plugins\",\n)\ndef create_token(\n id, secret, expires_after, alls, databases, resources, debug, plugins_dir\n):\n \"\"\"\n Create a signed API token for the specified actor ID\n\n Example:\n\n datasette create-token root --secret mysecret\n\n To allow only \"view-database-download\" for all databases:\n\n \\b\n datasette create-token root --secret mysecret \\\\\n --all view-database-download\n\n To allow \"create-table\" against a specific database:\n\n \\b\n datasette create-token root --secret mysecret \\\\\n --database mydb create-table\n\n To allow \"insert-row\" against a specific table:\n\n \\b\n datasette create-token root --secret myscret \\\\\n --resource mydb mytable insert-row\n\n Restricted actions can be specified multiple times using\n multiple --all, --database, and --resource options.\n\n Add --debug to see a decoded version of the token.\n \"\"\"\n ds = Datasette(secret=secret, plugins_dir=plugins_dir)\n\n # Run ds.invoke_startup() in an event loop\n try:\n run_sync(ds.invoke_startup)\n except StartupError as e:\n raise click.ClickException(e.args[0])\n\n # Warn about any unknown actions\n actions = []\n actions.extend(alls)\n actions.extend([p[1] for p in databases])\n actions.extend([p[2] for p in resources])\n for action in actions:\n if not ds.actions.get(action):\n click.secho(\n f\" Unknown permission: {action} \",\n fg=\"red\",\n err=True,\n )\n\n from datasette.tokens import TokenRestrictions\n\n restrictions = TokenRestrictions()\n for action in alls:\n restrictions.allow_all(action)\n for database, action in databases:\n restrictions.allow_database(database, action)\n for database, resource, action in resources:\n restrictions.allow_resource(database, resource, action)\n\n token = run_sync(\n lambda: ds.create_token(\n id,\n expires_after=expires_after,\n restrictions=restrictions,\n handler=\"signed\",\n )\n )\n click.echo(token)\n if debug:\n encoded = token[len(\"dstok_\") :]\n click.echo(\"\\nDecoded:\\n\")\n click.echo(json.dumps(ds.unsign(encoded, namespace=\"token\"), indent=2))\n\n\npm.hook.register_commands(cli=cli)\n\n\nasync def check_databases(ds):\n # Run check_connection against every connected database\n # to confirm they are all usable\n for database in list(ds.databases.values()):\n try:\n await database.execute_fn(check_connection)\n except SpatialiteConnectionProblem:\n suggestion = \"\"\n try:\n find_spatialite()\n suggestion = \"\\n\\nTry adding the --load-extension=spatialite option.\"\n except SpatialiteNotFound:\n pass\n raise click.UsageError(\n \"It looks like you're trying to load a SpatiaLite\"\n + \" database without first loading the SpatiaLite module.\"\n + suggestion\n + \"\\n\\nRead more: https://docs.datasette.io/en/stable/spatialite.html\"\n )\n except ConnectionProblem as e:\n raise click.UsageError(\n f\"Connection to {database.path} failed check: {str(e.args[0])}\"\n )\n # If --crossdb and more than SQLITE_LIMIT_ATTACHED show warning\n if (\n ds.crossdb\n and len([db for db in ds.databases.values() if not db.is_memory])\n > SQLITE_LIMIT_ATTACHED\n ):\n msg = (\n \"Warning: --crossdb only works with the first {} attached databases\".format(\n SQLITE_LIMIT_ATTACHED\n )\n )\n click.echo(click.style(msg, bold=True, fg=\"yellow\"), err=True)\n" }, { "path": "datasette/column_types.py", "content": "from enum import Enum\n\n\nclass SQLiteType(Enum):\n TEXT = \"TEXT\"\n INTEGER = \"INTEGER\"\n REAL = \"REAL\"\n BLOB = \"BLOB\"\n NULL = \"NULL\"\n\n @classmethod\n def from_declared_type(cls, declared_type: str | None) -> \"SQLiteType | None\":\n if declared_type is None:\n return cls.NULL\n\n normalized = declared_type.strip().upper()\n if not normalized:\n return cls.NULL\n\n if normalized == cls.NULL.value:\n return cls.NULL\n if \"INT\" in normalized:\n return cls.INTEGER\n if any(token in normalized for token in (\"CHAR\", \"CLOB\", \"TEXT\")):\n return cls.TEXT\n if \"BLOB\" in normalized:\n return cls.BLOB\n if any(\n token in normalized\n for token in (\"REAL\", \"FLOA\", \"DOUB\") # codespell:ignore doub\n ):\n return cls.REAL\n\n return None\n\n\nclass ColumnType:\n \"\"\"\n Base class for column types.\n\n Subclasses must define ``name`` and ``description`` as class attributes:\n\n - ``name``: Unique identifier string. Lowercase, no spaces.\n Examples: \"markdown\", \"file\", \"email\", \"url\", \"point\", \"image\".\n - ``description``: Human-readable label for admin UI dropdowns.\n Examples: \"Markdown text\", \"File reference\", \"Email address\".\n - ``sqlite_types``: Optional tuple of SQLiteType values restricting\n which SQLite column types this ColumnType can be assigned to.\n\n Instantiate with an optional ``config`` dict to bind per-column\n configuration::\n\n ct = MyColumnType(config={\"key\": \"value\"})\n ct.config # {\"key\": \"value\"}\n \"\"\"\n\n name: str\n description: str\n sqlite_types: tuple[SQLiteType, ...] | None = None\n\n def __init__(self, config=None):\n self.config = config\n\n async def render_cell(self, value, column, table, database, datasette, request):\n \"\"\"\n Return an HTML string to render this cell value, or None to\n fall through to the default render_cell plugin hook chain.\n \"\"\"\n return None\n\n async def validate(self, value, datasette):\n \"\"\"\n Validate a value before it is written. Return None if valid,\n or a string error message if invalid.\n \"\"\"\n return None\n\n async def transform_value(self, value, datasette):\n \"\"\"\n Transform a value before it appears in JSON API output.\n Return the transformed value. Default: return unchanged.\n \"\"\"\n return value\n" }, { "path": "datasette/database.py", "content": "import asyncio\nfrom collections import namedtuple\nfrom pathlib import Path\nimport janus\nimport queue\nimport sqlite_utils\nimport sys\nimport threading\nimport uuid\n\nfrom .tracer import trace\nfrom .utils import (\n detect_fts,\n detect_primary_keys,\n detect_spatialite,\n get_all_foreign_keys,\n get_outbound_foreign_keys,\n md5_not_usedforsecurity,\n sqlite_timelimit,\n sqlite3,\n table_columns,\n table_column_details,\n)\nfrom .utils.sqlite import sqlite_version\nfrom .inspect import inspect_hash\n\nconnections = threading.local()\n\nAttachedDatabase = namedtuple(\"AttachedDatabase\", (\"seq\", \"name\", \"file\"))\n\n\nclass Database:\n # For table counts stop at this many rows:\n count_limit = 10000\n _thread_local_id_counter = 1\n\n def __init__(\n self,\n ds,\n path=None,\n is_mutable=True,\n is_memory=False,\n memory_name=None,\n mode=None,\n ):\n self.name = None\n self._thread_local_id = f\"x{self._thread_local_id_counter}\"\n Database._thread_local_id_counter += 1\n self.route = None\n self.ds = ds\n self.path = path\n self.is_mutable = is_mutable\n self.is_memory = is_memory\n self.memory_name = memory_name\n if memory_name is not None:\n self.is_memory = True\n self.cached_hash = None\n self.cached_size = None\n self._cached_table_counts = None\n self._write_thread = None\n self._write_queue = None\n # These are used when in non-threaded mode:\n self._read_connection = None\n self._write_connection = None\n # This is used to track all file connections so they can be closed\n self._all_file_connections = []\n self.mode = mode\n\n @property\n def cached_table_counts(self):\n if self._cached_table_counts is not None:\n return self._cached_table_counts\n # Maybe use self.ds.inspect_data to populate cached_table_counts\n if self.ds.inspect_data and self.ds.inspect_data.get(self.name):\n self._cached_table_counts = {\n key: value[\"count\"]\n for key, value in self.ds.inspect_data[self.name][\"tables\"].items()\n }\n return self._cached_table_counts\n\n @property\n def color(self):\n if self.hash:\n return self.hash[:6]\n return md5_not_usedforsecurity(self.name)[:6]\n\n def suggest_name(self):\n if self.path:\n return Path(self.path).stem\n elif self.memory_name:\n return self.memory_name\n else:\n return \"db\"\n\n def connect(self, write=False):\n extra_kwargs = {}\n if write:\n extra_kwargs[\"isolation_level\"] = \"IMMEDIATE\"\n if self.memory_name:\n uri = \"file:{}?mode=memory&cache=shared\".format(self.memory_name)\n conn = sqlite3.connect(\n uri, uri=True, check_same_thread=False, **extra_kwargs\n )\n if not write:\n conn.execute(\"PRAGMA query_only=1\")\n return conn\n if self.is_memory:\n return sqlite3.connect(\":memory:\", uri=True)\n\n # mode=ro or immutable=1?\n if self.is_mutable:\n qs = \"?mode=ro\"\n if self.ds.nolock:\n qs += \"&nolock=1\"\n else:\n qs = \"?immutable=1\"\n assert not (write and not self.is_mutable)\n if write:\n qs = \"\"\n if self.mode is not None:\n qs = f\"?mode={self.mode}\"\n conn = sqlite3.connect(\n f\"file:{self.path}{qs}\", uri=True, check_same_thread=False, **extra_kwargs\n )\n self._all_file_connections.append(conn)\n return conn\n\n def close(self):\n # Close all connections - useful to avoid running out of file handles in tests\n for connection in self._all_file_connections:\n connection.close()\n\n async def execute_write(self, sql, params=None, block=True, request=None):\n def _inner(conn):\n return conn.execute(sql, params or [])\n\n with trace(\"sql\", database=self.name, sql=sql.strip(), params=params):\n results = await self.execute_write_fn(_inner, block=block, request=request)\n return results\n\n async def execute_write_script(self, sql, block=True, request=None):\n def _inner(conn):\n return conn.executescript(sql)\n\n with trace(\"sql\", database=self.name, sql=sql.strip(), executescript=True):\n results = await self.execute_write_fn(\n _inner, block=block, transaction=False, request=request\n )\n return results\n\n async def execute_write_many(self, sql, params_seq, block=True, request=None):\n def _inner(conn):\n count = 0\n\n def count_params(params):\n nonlocal count\n for param in params:\n count += 1\n yield param\n\n return conn.executemany(sql, count_params(params_seq)), count\n\n with trace(\n \"sql\", database=self.name, sql=sql.strip(), executemany=True\n ) as kwargs:\n results, count = await self.execute_write_fn(\n _inner, block=block, request=request\n )\n kwargs[\"count\"] = count\n return results\n\n async def execute_isolated_fn(self, fn):\n # Open a new connection just for the duration of this function\n # blocking the write queue to avoid any writes occurring during it\n if self.ds.executor is None:\n # non-threaded mode\n isolated_connection = self.connect(write=True)\n try:\n result = fn(isolated_connection)\n finally:\n isolated_connection.close()\n try:\n self._all_file_connections.remove(isolated_connection)\n except ValueError:\n # Was probably a memory connection\n pass\n return result\n else:\n # Threaded mode - send to write thread\n return await self._send_to_write_thread(fn, isolated_connection=True)\n\n async def execute_write_fn(self, fn, block=True, transaction=True, request=None):\n fn = self._wrap_fn_with_hooks(fn, request, transaction)\n if self.ds.executor is None:\n # non-threaded mode\n if self._write_connection is None:\n self._write_connection = self.connect(write=True)\n self.ds._prepare_connection(self._write_connection, self.name)\n if transaction:\n with self._write_connection:\n return fn(self._write_connection)\n else:\n return fn(self._write_connection)\n else:\n return await self._send_to_write_thread(\n fn, block=block, transaction=transaction\n )\n\n def _wrap_fn_with_hooks(self, fn, request, transaction):\n from .plugins import pm\n\n wrappers = pm.hook.write_wrapper(\n datasette=self.ds,\n database=self.name,\n request=request,\n transaction=transaction,\n )\n wrappers = [w for w in wrappers if w is not None]\n if not wrappers:\n return fn\n # Build the wrapped fn by nesting context manager generators.\n # The first wrapper returned by pluggy is outermost.\n original_fn = fn\n for wrapper_factory in reversed(wrappers):\n original_fn = _apply_write_wrapper(original_fn, wrapper_factory)\n return original_fn\n\n async def _send_to_write_thread(\n self, fn, block=True, isolated_connection=False, transaction=True\n ):\n if self._write_queue is None:\n self._write_queue = queue.Queue()\n if self._write_thread is None:\n self._write_thread = threading.Thread(\n target=self._execute_writes, daemon=True\n )\n self._write_thread.name = \"_execute_writes for database {}\".format(\n self.name\n )\n self._write_thread.start()\n task_id = uuid.uuid5(uuid.NAMESPACE_DNS, \"datasette.io\")\n reply_queue = janus.Queue()\n self._write_queue.put(\n WriteTask(fn, task_id, reply_queue, isolated_connection, transaction)\n )\n if block:\n result = await reply_queue.async_q.get()\n if isinstance(result, Exception):\n raise result\n else:\n return result\n else:\n return task_id\n\n def _execute_writes(self):\n # Infinite looping thread that protects the single write connection\n # to this database\n conn_exception = None\n conn = None\n try:\n conn = self.connect(write=True)\n self.ds._prepare_connection(conn, self.name)\n except Exception as e:\n conn_exception = e\n while True:\n task = self._write_queue.get()\n if conn_exception is not None:\n result = conn_exception\n else:\n if task.isolated_connection:\n isolated_connection = self.connect(write=True)\n try:\n result = task.fn(isolated_connection)\n except Exception as e:\n sys.stderr.write(\"{}\\n\".format(e))\n sys.stderr.flush()\n result = e\n finally:\n isolated_connection.close()\n try:\n self._all_file_connections.remove(isolated_connection)\n except ValueError:\n # Was probably a memory connection\n pass\n else:\n try:\n if task.transaction:\n with conn:\n result = task.fn(conn)\n else:\n result = task.fn(conn)\n except Exception as e:\n sys.stderr.write(\"{}\\n\".format(e))\n sys.stderr.flush()\n result = e\n task.reply_queue.sync_q.put(result)\n\n async def execute_fn(self, fn):\n if self.ds.executor is None:\n # non-threaded mode\n if self._read_connection is None:\n self._read_connection = self.connect()\n self.ds._prepare_connection(self._read_connection, self.name)\n return fn(self._read_connection)\n\n # threaded mode\n def in_thread():\n conn = getattr(connections, self._thread_local_id, None)\n if not conn:\n conn = self.connect()\n self.ds._prepare_connection(conn, self.name)\n setattr(connections, self._thread_local_id, conn)\n return fn(conn)\n\n return await asyncio.get_event_loop().run_in_executor(\n self.ds.executor, in_thread\n )\n\n async def execute(\n self,\n sql,\n params=None,\n truncate=False,\n custom_time_limit=None,\n page_size=None,\n log_sql_errors=True,\n ):\n \"\"\"Executes sql against db_name in a thread\"\"\"\n page_size = page_size or self.ds.page_size\n\n def sql_operation_in_thread(conn):\n time_limit_ms = self.ds.sql_time_limit_ms\n if custom_time_limit and custom_time_limit < time_limit_ms:\n time_limit_ms = custom_time_limit\n\n with sqlite_timelimit(conn, time_limit_ms):\n try:\n cursor = conn.cursor()\n cursor.execute(sql, params if params is not None else {})\n max_returned_rows = self.ds.max_returned_rows\n if max_returned_rows == page_size:\n max_returned_rows += 1\n if max_returned_rows and truncate:\n rows = cursor.fetchmany(max_returned_rows + 1)\n truncated = len(rows) > max_returned_rows\n rows = rows[:max_returned_rows]\n else:\n rows = cursor.fetchall()\n truncated = False\n except (sqlite3.OperationalError, sqlite3.DatabaseError) as e:\n if e.args == (\"interrupted\",):\n raise QueryInterrupted(e, sql, params)\n if log_sql_errors:\n sys.stderr.write(\n \"ERROR: conn={}, sql = {}, params = {}: {}\\n\".format(\n conn, repr(sql), params, e\n )\n )\n sys.stderr.flush()\n raise\n\n if truncate:\n return Results(rows, truncated, cursor.description)\n\n else:\n return Results(rows, False, cursor.description)\n\n with trace(\"sql\", database=self.name, sql=sql.strip(), params=params):\n results = await self.execute_fn(sql_operation_in_thread)\n return results\n\n @property\n def hash(self):\n if self.cached_hash is not None:\n return self.cached_hash\n elif self.is_mutable or self.is_memory:\n return None\n elif self.ds.inspect_data and self.ds.inspect_data.get(self.name):\n self.cached_hash = self.ds.inspect_data[self.name][\"hash\"]\n return self.cached_hash\n else:\n p = Path(self.path)\n self.cached_hash = inspect_hash(p)\n return self.cached_hash\n\n @property\n def size(self):\n if self.cached_size is not None:\n return self.cached_size\n elif self.is_memory:\n return 0\n elif self.is_mutable:\n return Path(self.path).stat().st_size\n elif self.ds.inspect_data and self.ds.inspect_data.get(self.name):\n self.cached_size = self.ds.inspect_data[self.name][\"size\"]\n return self.cached_size\n else:\n self.cached_size = Path(self.path).stat().st_size\n return self.cached_size\n\n async def table_counts(self, limit=10):\n if not self.is_mutable and self.cached_table_counts is not None:\n return self.cached_table_counts\n # Try to get counts for each table, $limit timeout for each count\n counts = {}\n for table in await self.table_names():\n try:\n table_count = (\n await self.execute(\n f\"select count(*) from (select * from [{table}] limit {self.count_limit + 1})\",\n custom_time_limit=limit,\n )\n ).rows[0][0]\n counts[table] = table_count\n # In some cases I saw \"SQL Logic Error\" here in addition to\n # QueryInterrupted - so we catch that too:\n except (QueryInterrupted, sqlite3.OperationalError, sqlite3.DatabaseError):\n counts[table] = None\n if not self.is_mutable:\n self._cached_table_counts = counts\n return counts\n\n @property\n def mtime_ns(self):\n if self.is_memory:\n return None\n return Path(self.path).stat().st_mtime_ns\n\n async def attached_databases(self):\n # This used to be:\n # select seq, name, file from pragma_database_list() where seq > 0\n # But SQLite prior to 3.16.0 doesn't support pragma functions\n results = await self.execute(\"PRAGMA database_list;\")\n # {'seq': 0, 'name': 'main', 'file': ''}\n return [\n AttachedDatabase(*row)\n for row in results.rows\n # Filter out the SQLite internal \"temp\" database, refs #2557\n if row[\"seq\"] > 0 and row[\"name\"] != \"temp\"\n ]\n\n async def table_exists(self, table):\n results = await self.execute(\n \"select 1 from sqlite_master where type='table' and name=?\", params=(table,)\n )\n return bool(results.rows)\n\n async def view_exists(self, table):\n results = await self.execute(\n \"select 1 from sqlite_master where type='view' and name=?\", params=(table,)\n )\n return bool(results.rows)\n\n async def table_names(self):\n results = await self.execute(\n \"select name from sqlite_master where type='table' order by name\"\n )\n return [r[0] for r in results.rows]\n\n async def table_columns(self, table):\n return await self.execute_fn(lambda conn: table_columns(conn, table))\n\n async def table_column_details(self, table):\n return await self.execute_fn(lambda conn: table_column_details(conn, table))\n\n async def primary_keys(self, table):\n return await self.execute_fn(lambda conn: detect_primary_keys(conn, table))\n\n async def fts_table(self, table):\n return await self.execute_fn(lambda conn: detect_fts(conn, table))\n\n async def label_column_for_table(self, table):\n explicit_label_column = (await self.ds.table_config(self.name, table)).get(\n \"label_column\"\n )\n if explicit_label_column:\n return explicit_label_column\n\n def column_details(conn):\n # Returns {column_name: (type, is_unique)}\n db = sqlite_utils.Database(conn)\n columns = db[table].columns_dict\n indexes = db[table].indexes\n details = {}\n for name in columns:\n is_unique = any(\n index\n for index in indexes\n if index.columns == [name] and index.unique\n )\n details[name] = (columns[name], is_unique)\n return details\n\n column_details = await self.execute_fn(column_details)\n # Is there just one unique column that's text?\n unique_text_columns = [\n name\n for name, (type_, is_unique) in column_details.items()\n if is_unique and type_ is str\n ]\n if len(unique_text_columns) == 1:\n return unique_text_columns[0]\n\n column_names = list(column_details.keys())\n # Is there a name or title column?\n name_or_title = [c for c in column_names if c.lower() in (\"name\", \"title\")]\n if name_or_title:\n return name_or_title[0]\n # If a table has two columns, one of which is ID, then label_column is the other one\n if (\n column_names\n and len(column_names) == 2\n and (\"id\" in column_names or \"pk\" in column_names)\n and not set(column_names) == {\"id\", \"pk\"}\n ):\n return [c for c in column_names if c not in (\"id\", \"pk\")][0]\n # Couldn't find a label:\n return None\n\n async def foreign_keys_for_table(self, table):\n return await self.execute_fn(\n lambda conn: get_outbound_foreign_keys(conn, table)\n )\n\n async def hidden_table_names(self):\n hidden_tables = []\n # Add any tables marked as hidden in config\n db_config = self.ds.config.get(\"databases\", {}).get(self.name, {})\n if \"tables\" in db_config:\n hidden_tables += [\n t for t in db_config[\"tables\"] if db_config[\"tables\"][t].get(\"hidden\")\n ]\n\n if sqlite_version()[1] >= 37:\n hidden_tables += [x[0] for x in await self.execute(\"\"\"\n with shadow_tables as (\n select name\n from pragma_table_list\n where [type] = 'shadow'\n order by name\n ),\n core_tables as (\n select name\n from sqlite_master\n WHERE name in ('sqlite_stat1', 'sqlite_stat2', 'sqlite_stat3', 'sqlite_stat4')\n OR substr(name, 1, 1) == '_'\n ),\n combined as (\n select name from shadow_tables\n union all\n select name from core_tables\n )\n select name from combined order by 1\n \"\"\")]\n else:\n hidden_tables += [x[0] for x in await self.execute(\"\"\"\n WITH base AS (\n SELECT name\n FROM sqlite_master\n WHERE name IN ('sqlite_stat1', 'sqlite_stat2', 'sqlite_stat3', 'sqlite_stat4')\n OR substr(name, 1, 1) == '_'\n ),\n fts_suffixes AS (\n SELECT column1 AS suffix\n FROM (VALUES ('_data'), ('_idx'), ('_docsize'), ('_content'), ('_config'))\n ),\n fts5_names AS (\n SELECT name\n FROM sqlite_master\n WHERE sql LIKE '%VIRTUAL TABLE%USING FTS%'\n ),\n fts5_shadow_tables AS (\n SELECT\n printf('%s%s', fts5_names.name, fts_suffixes.suffix) AS name\n FROM fts5_names\n JOIN fts_suffixes\n ),\n fts3_suffixes AS (\n SELECT column1 AS suffix\n FROM (VALUES ('_content'), ('_segdir'), ('_segments'), ('_stat'), ('_docsize'))\n ),\n fts3_names AS (\n SELECT name\n FROM sqlite_master\n WHERE sql LIKE '%VIRTUAL TABLE%USING FTS3%'\n OR sql LIKE '%VIRTUAL TABLE%USING FTS4%'\n ),\n fts3_shadow_tables AS (\n SELECT\n printf('%s%s', fts3_names.name, fts3_suffixes.suffix) AS name\n FROM fts3_names\n JOIN fts3_suffixes\n ),\n final AS (\n SELECT name FROM base\n UNION ALL\n SELECT name FROM fts5_shadow_tables\n UNION ALL\n SELECT name FROM fts3_shadow_tables\n )\n SELECT name FROM final ORDER BY 1\n \"\"\")]\n # Also hide any FTS tables that have a content= argument\n hidden_tables += [x[0] for x in await self.execute(\"\"\"\n SELECT name\n FROM sqlite_master\n WHERE sql LIKE '%VIRTUAL TABLE%'\n AND sql LIKE '%USING FTS%'\n AND sql LIKE '%content=%'\n \"\"\")]\n\n has_spatialite = await self.execute_fn(detect_spatialite)\n if has_spatialite:\n # Also hide Spatialite internal tables\n hidden_tables += [\n \"ElementaryGeometries\",\n \"SpatialIndex\",\n \"geometry_columns\",\n \"spatial_ref_sys\",\n \"spatialite_history\",\n \"sql_statements_log\",\n \"sqlite_sequence\",\n \"views_geometry_columns\",\n \"virts_geometry_columns\",\n \"data_licenses\",\n \"KNN\",\n \"KNN2\",\n ] + [\n r[0] for r in (await self.execute(\"\"\"\n select name from sqlite_master\n where name like \"idx_%\"\n and type = \"table\"\n \"\"\")).rows\n ]\n\n return hidden_tables\n\n async def view_names(self):\n results = await self.execute(\"select name from sqlite_master where type='view'\")\n return [r[0] for r in results.rows]\n\n async def get_all_foreign_keys(self):\n return await self.execute_fn(get_all_foreign_keys)\n\n async def get_table_definition(self, table, type_=\"table\"):\n table_definition_rows = list(\n await self.execute(\n \"select sql from sqlite_master where name = :n and type=:t\",\n {\"n\": table, \"t\": type_},\n )\n )\n if not table_definition_rows:\n return None\n bits = [table_definition_rows[0][0] + \";\"]\n # Add on any indexes\n index_rows = list(\n await self.execute(\n \"select sql from sqlite_master where tbl_name = :n and type='index' and sql is not null\",\n {\"n\": table},\n )\n )\n for index_row in index_rows:\n bits.append(index_row[0] + \";\")\n return \"\\n\".join(bits)\n\n async def get_view_definition(self, view):\n return await self.get_table_definition(view, \"view\")\n\n def __repr__(self):\n tags = []\n if self.is_mutable:\n tags.append(\"mutable\")\n if self.is_memory:\n tags.append(\"memory\")\n if self.hash:\n tags.append(f\"hash={self.hash}\")\n if self.size is not None:\n tags.append(f\"size={self.size}\")\n tags_str = \"\"\n if tags:\n tags_str = f\" ({', '.join(tags)})\"\n return f\"\"\n\n\ndef _apply_write_wrapper(fn, wrapper_factory):\n \"\"\"Apply a single write_wrapper context manager around fn.\n\n ``wrapper_factory`` is a callable that takes ``(conn)`` and returns a\n generator that yields exactly once. Code before the yield runs before\n ``fn(conn)``, code after the yield runs after. The result of\n ``fn(conn)`` is sent into the generator via ``.send()``, and any\n exception raised by ``fn(conn)`` is thrown via ``.throw()``.\n \"\"\"\n\n def wrapped(conn):\n gen = wrapper_factory(conn)\n # Advance to the yield point (run \"before\" code)\n try:\n next(gen)\n except StopIteration:\n # Generator didn't yield — just run fn unchanged\n return fn(conn)\n\n # Execute the actual write\n try:\n result = fn(conn)\n except Exception:\n # Throw exception into generator so it can handle it\n try:\n gen.throw(*sys.exc_info())\n except StopIteration:\n pass\n # Re-raise the original exception\n raise\n else:\n # Send the result back through the yield\n try:\n gen.send(result)\n except StopIteration:\n pass\n return result\n\n return wrapped\n\n\nclass WriteTask:\n __slots__ = (\"fn\", \"task_id\", \"reply_queue\", \"isolated_connection\", \"transaction\")\n\n def __init__(self, fn, task_id, reply_queue, isolated_connection, transaction):\n self.fn = fn\n self.task_id = task_id\n self.reply_queue = reply_queue\n self.isolated_connection = isolated_connection\n self.transaction = transaction\n\n\nclass QueryInterrupted(Exception):\n def __init__(self, e, sql, params):\n self.e = e\n self.sql = sql\n self.params = params\n\n def __str__(self):\n return \"QueryInterrupted: {}\".format(self.e)\n\n\nclass MultipleValues(Exception):\n pass\n\n\nclass Results:\n def __init__(self, rows, truncated, description):\n self.rows = rows\n self.truncated = truncated\n self.description = description\n\n @property\n def columns(self):\n return [d[0] for d in self.description]\n\n def first(self):\n if self.rows:\n return self.rows[0]\n else:\n return None\n\n def single_value(self):\n if self.rows and 1 == len(self.rows) and 1 == len(self.rows[0]):\n return self.rows[0][0]\n else:\n raise MultipleValues\n\n def dicts(self):\n return [dict(row) for row in self.rows]\n\n def __iter__(self):\n return iter(self.rows)\n\n def __len__(self):\n return len(self.rows)\n" }, { "path": "datasette/default_actions.py", "content": "from datasette import hookimpl\nfrom datasette.permissions import Action\nfrom datasette.resources import (\n DatabaseResource,\n TableResource,\n QueryResource,\n)\n\n\n@hookimpl\ndef register_actions():\n \"\"\"Register the core Datasette actions.\"\"\"\n return (\n # Global actions (no resource_class)\n Action(\n name=\"view-instance\",\n abbr=\"vi\",\n description=\"View Datasette instance\",\n ),\n Action(\n name=\"permissions-debug\",\n abbr=\"pd\",\n description=\"Access permission debug tool\",\n ),\n Action(\n name=\"debug-menu\",\n abbr=\"dm\",\n description=\"View debug menu items\",\n ),\n # Database-level actions (parent-level)\n Action(\n name=\"view-database\",\n abbr=\"vd\",\n description=\"View database\",\n resource_class=DatabaseResource,\n ),\n Action(\n name=\"view-database-download\",\n abbr=\"vdd\",\n description=\"Download database file\",\n resource_class=DatabaseResource,\n also_requires=\"view-database\",\n ),\n Action(\n name=\"execute-sql\",\n abbr=\"es\",\n description=\"Execute read-only SQL queries\",\n resource_class=DatabaseResource,\n also_requires=\"view-database\",\n ),\n Action(\n name=\"create-table\",\n abbr=\"ct\",\n description=\"Create tables\",\n resource_class=DatabaseResource,\n ),\n # Table-level actions (child-level)\n Action(\n name=\"view-table\",\n abbr=\"vt\",\n description=\"View table\",\n resource_class=TableResource,\n ),\n Action(\n name=\"insert-row\",\n abbr=\"ir\",\n description=\"Insert rows\",\n resource_class=TableResource,\n ),\n Action(\n name=\"delete-row\",\n abbr=\"dr\",\n description=\"Delete rows\",\n resource_class=TableResource,\n ),\n Action(\n name=\"update-row\",\n abbr=\"ur\",\n description=\"Update rows\",\n resource_class=TableResource,\n ),\n Action(\n name=\"alter-table\",\n abbr=\"at\",\n description=\"Alter tables\",\n resource_class=TableResource,\n ),\n Action(\n name=\"set-column-type\",\n abbr=\"sct\",\n description=\"Set column type\",\n resource_class=TableResource,\n ),\n Action(\n name=\"drop-table\",\n abbr=\"dt\",\n description=\"Drop tables\",\n resource_class=TableResource,\n ),\n # Query-level actions (child-level)\n Action(\n name=\"view-query\",\n abbr=\"vq\",\n description=\"View named query results\",\n resource_class=QueryResource,\n ),\n )\n" }, { "path": "datasette/default_column_types.py", "content": "import json\nimport re\n\nimport markupsafe\n\nfrom datasette import hookimpl\nfrom datasette.column_types import ColumnType, SQLiteType\n\n\nclass UrlColumnType(ColumnType):\n name = \"url\"\n description = \"URL\"\n sqlite_types = (SQLiteType.TEXT,)\n\n async def render_cell(self, value, column, table, database, datasette, request):\n if not value or not isinstance(value, str):\n return None\n escaped = markupsafe.escape(value.strip())\n return markupsafe.Markup(f'{escaped}')\n\n async def validate(self, value, datasette):\n if value is None or value == \"\":\n return None\n if not isinstance(value, str):\n return \"URL must be a string\"\n if not re.match(r\"^https?://\\S+$\", value.strip()):\n return \"Invalid URL\"\n return None\n\n\nclass EmailColumnType(ColumnType):\n name = \"email\"\n description = \"Email address\"\n sqlite_types = (SQLiteType.TEXT,)\n\n async def render_cell(self, value, column, table, database, datasette, request):\n if not value or not isinstance(value, str):\n return None\n escaped = markupsafe.escape(value.strip())\n return markupsafe.Markup(f'{escaped}')\n\n async def validate(self, value, datasette):\n if value is None or value == \"\":\n return None\n if not isinstance(value, str):\n return \"Email must be a string\"\n if not re.match(r\"^[^@\\s]+@[^@\\s]+\\.[^@\\s]+$\", value.strip()):\n return \"Invalid email address\"\n return None\n\n\nclass JsonColumnType(ColumnType):\n name = \"json\"\n description = \"JSON data\"\n sqlite_types = (SQLiteType.TEXT,)\n\n async def render_cell(self, value, column, table, database, datasette, request):\n if value is None:\n return None\n try:\n parsed = json.loads(value) if isinstance(value, str) else value\n formatted = json.dumps(parsed, indent=2)\n escaped = markupsafe.escape(formatted)\n return markupsafe.Markup(f\"
{escaped}
\")\n except (json.JSONDecodeError, TypeError):\n return None\n\n async def validate(self, value, datasette):\n if value is None or value == \"\":\n return None\n if isinstance(value, str):\n try:\n json.loads(value)\n except json.JSONDecodeError:\n return \"Invalid JSON\"\n return None\n\n\n@hookimpl\ndef register_column_types(datasette):\n return [UrlColumnType, EmailColumnType, JsonColumnType]\n" }, { "path": "datasette/default_magic_parameters.py", "content": "from datasette import hookimpl\nimport datetime\nimport os\nimport time\n\n\ndef header(key, request):\n key = key.replace(\"_\", \"-\").encode(\"utf-8\")\n headers_dict = dict(request.scope[\"headers\"])\n return headers_dict.get(key, b\"\").decode(\"utf-8\")\n\n\ndef actor(key, request):\n if request.actor is None:\n raise KeyError\n return request.actor[key]\n\n\ndef cookie(key, request):\n return request.cookies[key]\n\n\ndef now(key, request):\n if key == \"epoch\":\n return int(time.time())\n elif key == \"date_utc\":\n return datetime.datetime.now(datetime.timezone.utc).date().isoformat()\n elif key == \"datetime_utc\":\n return (\n datetime.datetime.now(datetime.timezone.utc).strftime(r\"%Y-%m-%dT%H:%M:%S\")\n + \"Z\"\n )\n else:\n raise KeyError\n\n\ndef random(key, request):\n if key.startswith(\"chars_\") and key.split(\"chars_\")[-1].isdigit():\n num_chars = int(key.split(\"chars_\")[-1])\n if num_chars % 2 == 1:\n urandom_len = (num_chars + 1) / 2\n else:\n urandom_len = num_chars / 2\n return os.urandom(int(urandom_len)).hex()[:num_chars]\n else:\n raise KeyError\n\n\n@hookimpl\ndef register_magic_parameters():\n return [\n (\"header\", header),\n (\"actor\", actor),\n (\"cookie\", cookie),\n (\"now\", now),\n (\"random\", random),\n ]\n" }, { "path": "datasette/default_menu_links.py", "content": "from datasette import hookimpl\n\n\n@hookimpl\ndef menu_links(datasette, actor):\n async def inner():\n if not await datasette.allowed(action=\"debug-menu\", actor=actor):\n return []\n\n return [\n {\"href\": datasette.urls.path(\"/-/databases\"), \"label\": \"Databases\"},\n {\n \"href\": datasette.urls.path(\"/-/plugins\"),\n \"label\": \"Installed plugins\",\n },\n {\n \"href\": datasette.urls.path(\"/-/versions\"),\n \"label\": \"Version info\",\n },\n {\n \"href\": datasette.urls.path(\"/-/settings\"),\n \"label\": \"Settings\",\n },\n {\n \"href\": datasette.urls.path(\"/-/permissions\"),\n \"label\": \"Debug permissions\",\n },\n {\n \"href\": datasette.urls.path(\"/-/messages\"),\n \"label\": \"Debug messages\",\n },\n {\n \"href\": datasette.urls.path(\"/-/allow-debug\"),\n \"label\": \"Debug allow rules\",\n },\n {\"href\": datasette.urls.path(\"/-/threads\"), \"label\": \"Debug threads\"},\n {\"href\": datasette.urls.path(\"/-/actor\"), \"label\": \"Debug actor\"},\n {\"href\": datasette.urls.path(\"/-/patterns\"), \"label\": \"Pattern portfolio\"},\n ]\n\n return inner\n" }, { "path": "datasette/default_permissions/__init__.py", "content": "\"\"\"\nDefault permission implementations for Datasette.\n\nThis module provides the built-in permission checking logic through implementations\nof the permission_resources_sql hook. The hooks are organized by their purpose:\n\n1. Actor Restrictions - Enforces _r allowlists embedded in actor tokens\n2. Root User - Grants full access when --root flag is used\n3. Config Rules - Applies permissions from datasette.yaml\n4. Default Settings - Enforces default_allow_sql and default view permissions\n\nIMPORTANT: These hooks return PermissionSQL objects that are combined using SQL\nUNION/INTERSECT operations. The order of evaluation is:\n - restriction_sql fields are INTERSECTed (all must match)\n - Regular sql fields are UNIONed and evaluated with cascading priority\n\"\"\"\n\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING, Optional\n\nif TYPE_CHECKING:\n from datasette.app import Datasette\n\nfrom datasette import hookimpl\n\n# Re-export all hooks and public utilities\nfrom .restrictions import (\n actor_restrictions_sql as actor_restrictions_sql,\n restrictions_allow_action as restrictions_allow_action,\n ActorRestrictions as ActorRestrictions,\n)\nfrom .root import root_user_permissions_sql as root_user_permissions_sql\nfrom .config import config_permissions_sql as config_permissions_sql\nfrom .defaults import (\n default_allow_sql_check as default_allow_sql_check,\n default_action_permissions_sql as default_action_permissions_sql,\n DEFAULT_ALLOW_ACTIONS as DEFAULT_ALLOW_ACTIONS,\n)\n\n\n@hookimpl\ndef skip_csrf(scope) -> Optional[bool]:\n \"\"\"Skip CSRF check for JSON content-type requests.\"\"\"\n if scope[\"type\"] == \"http\":\n headers = scope.get(\"headers\") or {}\n if dict(headers).get(b\"content-type\") == b\"application/json\":\n return True\n return None\n\n\n@hookimpl\ndef canned_queries(datasette: \"Datasette\", database: str, actor) -> dict:\n \"\"\"Return canned queries defined in datasette.yaml configuration.\"\"\"\n queries = (\n ((datasette.config or {}).get(\"databases\") or {}).get(database) or {}\n ).get(\"queries\") or {}\n return queries\n" }, { "path": "datasette/default_permissions/config.py", "content": "\"\"\"\nConfig-based permission handling for Datasette.\n\nApplies permission rules from datasette.yaml configuration.\n\"\"\"\n\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING, Any, List, Optional, Set, Tuple\n\nif TYPE_CHECKING:\n from datasette.app import Datasette\n\nfrom datasette import hookimpl\nfrom datasette.permissions import PermissionSQL\nfrom datasette.utils import actor_matches_allow\n\nfrom .helpers import PermissionRowCollector, get_action_name_variants\n\n\nclass ConfigPermissionProcessor:\n \"\"\"\n Processes permission rules from datasette.yaml configuration.\n\n Configuration structure:\n\n permissions: # Root-level permissions block\n view-instance:\n id: admin\n\n databases:\n mydb:\n permissions: # Database-level permissions\n view-database:\n id: admin\n allow: # Database-level allow block (for view-*)\n id: viewer\n allow_sql: # execute-sql allow block\n id: analyst\n tables:\n users:\n permissions: # Table-level permissions\n view-table:\n id: admin\n allow: # Table-level allow block\n id: viewer\n queries:\n my_query:\n permissions: # Query-level permissions\n view-query:\n id: admin\n allow: # Query-level allow block\n id: viewer\n \"\"\"\n\n def __init__(\n self,\n datasette: \"Datasette\",\n actor: Optional[dict],\n action: str,\n ):\n self.datasette = datasette\n self.actor = actor\n self.action = action\n self.config = datasette.config or {}\n self.collector = PermissionRowCollector(prefix=\"cfg\")\n\n # Pre-compute action variants\n self.action_checks = get_action_name_variants(datasette, action)\n self.action_obj = datasette.actions.get(action)\n\n # Parse restrictions if present\n self.has_restrictions = actor and \"_r\" in actor if actor else False\n self.restrictions = actor.get(\"_r\", {}) if actor else {}\n\n # Pre-compute restriction info for efficiency\n self.restricted_databases: Set[str] = set()\n self.restricted_tables: Set[Tuple[str, str]] = set()\n\n if self.has_restrictions:\n self.restricted_databases = {\n db_name\n for db_name, db_actions in (self.restrictions.get(\"d\") or {}).items()\n if self.action_checks.intersection(db_actions)\n }\n self.restricted_tables = {\n (db_name, table_name)\n for db_name, tables in (self.restrictions.get(\"r\") or {}).items()\n for table_name, table_actions in tables.items()\n if self.action_checks.intersection(table_actions)\n }\n # Tables implicitly reference their parent databases\n self.restricted_databases.update(db for db, _ in self.restricted_tables)\n\n def evaluate_allow_block(self, allow_block: Any) -> Optional[bool]:\n \"\"\"Evaluate an allow block against the current actor.\"\"\"\n if allow_block is None:\n return None\n return actor_matches_allow(self.actor, allow_block)\n\n def is_in_restriction_allowlist(\n self,\n parent: Optional[str],\n child: Optional[str],\n ) -> bool:\n \"\"\"Check if resource is allowed by actor restrictions.\"\"\"\n if not self.has_restrictions:\n return True # No restrictions, all resources allowed\n\n # Check global allowlist\n if self.action_checks.intersection(self.restrictions.get(\"a\", [])):\n return True\n\n # Check database-level allowlist\n if parent and self.action_checks.intersection(\n self.restrictions.get(\"d\", {}).get(parent, [])\n ):\n return True\n\n # Check table-level allowlist\n if parent:\n table_restrictions = (self.restrictions.get(\"r\", {}) or {}).get(parent, {})\n if child:\n table_actions = table_restrictions.get(child, [])\n if self.action_checks.intersection(table_actions):\n return True\n else:\n # Parent query should proceed if any child in this database is allowlisted\n for table_actions in table_restrictions.values():\n if self.action_checks.intersection(table_actions):\n return True\n\n # Parent/child both None: include if any restrictions exist for this action\n if parent is None and child is None:\n if self.action_checks.intersection(self.restrictions.get(\"a\", [])):\n return True\n if self.restricted_databases:\n return True\n if self.restricted_tables:\n return True\n\n return False\n\n def add_permissions_rule(\n self,\n parent: Optional[str],\n child: Optional[str],\n permissions_block: Optional[dict],\n scope_desc: str,\n ) -> None:\n \"\"\"Add a rule from a permissions:{action} block.\"\"\"\n if permissions_block is None:\n return\n\n action_allow_block = permissions_block.get(self.action)\n result = self.evaluate_allow_block(action_allow_block)\n\n self.collector.add(\n parent=parent,\n child=child,\n allow=result,\n reason=f\"config {'allow' if result else 'deny'} {scope_desc}\",\n if_not_none=True,\n )\n\n def add_allow_block_rule(\n self,\n parent: Optional[str],\n child: Optional[str],\n allow_block: Any,\n scope_desc: str,\n ) -> None:\n \"\"\"\n Add rules from an allow:{} block.\n\n For allow blocks, if the block exists but doesn't match the actor,\n this is treated as a deny. We also handle the restriction-gate logic.\n \"\"\"\n if allow_block is None:\n return\n\n # Skip if resource is not in restriction allowlist\n if not self.is_in_restriction_allowlist(parent, child):\n return\n\n result = self.evaluate_allow_block(allow_block)\n bool_result = bool(result)\n\n self.collector.add(\n parent,\n child,\n bool_result,\n f\"config {'allow' if result else 'deny'} {scope_desc}\",\n )\n\n # Handle restriction-gate: add explicit denies for restricted resources\n self._add_restriction_gate_denies(parent, child, bool_result, scope_desc)\n\n def _add_restriction_gate_denies(\n self,\n parent: Optional[str],\n child: Optional[str],\n is_allowed: bool,\n scope_desc: str,\n ) -> None:\n \"\"\"\n When a config rule denies at a higher level, add explicit denies\n for restricted resources to prevent child-level allows from\n incorrectly granting access.\n \"\"\"\n if is_allowed or child is not None or not self.has_restrictions:\n return\n\n if not self.action_obj:\n return\n\n reason = f\"config deny {scope_desc} (restriction gate)\"\n\n if parent is None:\n # Root-level deny: add denies for all restricted resources\n if self.action_obj.takes_parent:\n for db_name in self.restricted_databases:\n self.collector.add(db_name, None, False, reason)\n if self.action_obj.takes_child:\n for db_name, table_name in self.restricted_tables:\n self.collector.add(db_name, table_name, False, reason)\n else:\n # Database-level deny: add denies for tables in that database\n if self.action_obj.takes_child:\n for db_name, table_name in self.restricted_tables:\n if db_name == parent:\n self.collector.add(db_name, table_name, False, reason)\n\n def process(self) -> Optional[PermissionSQL]:\n \"\"\"Process all config rules and return combined PermissionSQL.\"\"\"\n self._process_root_permissions()\n self._process_databases()\n self._process_root_allow_blocks()\n\n return self.collector.to_permission_sql()\n\n def _process_root_permissions(self) -> None:\n \"\"\"Process root-level permissions block.\"\"\"\n root_perms = self.config.get(\"permissions\") or {}\n self.add_permissions_rule(\n None,\n None,\n root_perms,\n f\"permissions for {self.action}\",\n )\n\n def _process_databases(self) -> None:\n \"\"\"Process database-level and nested configurations.\"\"\"\n databases = self.config.get(\"databases\") or {}\n\n for db_name, db_config in databases.items():\n self._process_database(db_name, db_config or {})\n\n def _process_database(self, db_name: str, db_config: dict) -> None:\n \"\"\"Process a single database's configuration.\"\"\"\n # Database-level permissions block\n db_perms = db_config.get(\"permissions\") or {}\n self.add_permissions_rule(\n db_name,\n None,\n db_perms,\n f\"permissions for {self.action} on {db_name}\",\n )\n\n # Process tables\n for table_name, table_config in (db_config.get(\"tables\") or {}).items():\n self._process_table(db_name, table_name, table_config or {})\n\n # Process queries\n for query_name, query_config in (db_config.get(\"queries\") or {}).items():\n self._process_query(db_name, query_name, query_config)\n\n # Database-level allow blocks\n self._process_database_allow_blocks(db_name, db_config)\n\n def _process_table(\n self,\n db_name: str,\n table_name: str,\n table_config: dict,\n ) -> None:\n \"\"\"Process a single table's configuration.\"\"\"\n # Table-level permissions block\n table_perms = table_config.get(\"permissions\") or {}\n self.add_permissions_rule(\n db_name,\n table_name,\n table_perms,\n f\"permissions for {self.action} on {db_name}/{table_name}\",\n )\n\n # Table-level allow block (for view-table)\n if self.action == \"view-table\":\n self.add_allow_block_rule(\n db_name,\n table_name,\n table_config.get(\"allow\"),\n f\"allow for {self.action} on {db_name}/{table_name}\",\n )\n\n def _process_query(\n self,\n db_name: str,\n query_name: str,\n query_config: Any,\n ) -> None:\n \"\"\"Process a single query's configuration.\"\"\"\n # Query config can be a string (just SQL) or dict\n if not isinstance(query_config, dict):\n return\n\n # Query-level permissions block\n query_perms = query_config.get(\"permissions\") or {}\n self.add_permissions_rule(\n db_name,\n query_name,\n query_perms,\n f\"permissions for {self.action} on {db_name}/{query_name}\",\n )\n\n # Query-level allow block (for view-query)\n if self.action == \"view-query\":\n self.add_allow_block_rule(\n db_name,\n query_name,\n query_config.get(\"allow\"),\n f\"allow for {self.action} on {db_name}/{query_name}\",\n )\n\n def _process_database_allow_blocks(\n self,\n db_name: str,\n db_config: dict,\n ) -> None:\n \"\"\"Process database-level allow/allow_sql blocks.\"\"\"\n # view-database allow block\n if self.action == \"view-database\":\n self.add_allow_block_rule(\n db_name,\n None,\n db_config.get(\"allow\"),\n f\"allow for {self.action} on {db_name}\",\n )\n\n # execute-sql allow_sql block\n if self.action == \"execute-sql\":\n self.add_allow_block_rule(\n db_name,\n None,\n db_config.get(\"allow_sql\"),\n f\"allow_sql for {db_name}\",\n )\n\n # view-table uses database-level allow for inheritance\n if self.action == \"view-table\":\n self.add_allow_block_rule(\n db_name,\n None,\n db_config.get(\"allow\"),\n f\"allow for {self.action} on {db_name}\",\n )\n\n # view-query uses database-level allow for inheritance\n if self.action == \"view-query\":\n self.add_allow_block_rule(\n db_name,\n None,\n db_config.get(\"allow\"),\n f\"allow for {self.action} on {db_name}\",\n )\n\n def _process_root_allow_blocks(self) -> None:\n \"\"\"Process root-level allow/allow_sql blocks.\"\"\"\n root_allow = self.config.get(\"allow\")\n\n if self.action == \"view-instance\":\n self.add_allow_block_rule(\n None,\n None,\n root_allow,\n \"allow for view-instance\",\n )\n\n if self.action == \"view-database\":\n self.add_allow_block_rule(\n None,\n None,\n root_allow,\n \"allow for view-database\",\n )\n\n if self.action == \"view-table\":\n self.add_allow_block_rule(\n None,\n None,\n root_allow,\n \"allow for view-table\",\n )\n\n if self.action == \"view-query\":\n self.add_allow_block_rule(\n None,\n None,\n root_allow,\n \"allow for view-query\",\n )\n\n if self.action == \"execute-sql\":\n self.add_allow_block_rule(\n None,\n None,\n self.config.get(\"allow_sql\"),\n \"allow_sql\",\n )\n\n\n@hookimpl(specname=\"permission_resources_sql\")\nasync def config_permissions_sql(\n datasette: \"Datasette\",\n actor: Optional[dict],\n action: str,\n) -> Optional[List[PermissionSQL]]:\n \"\"\"\n Apply permission rules from datasette.yaml configuration.\n\n This processes:\n - permissions: blocks at root, database, table, and query levels\n - allow: blocks for view-* actions\n - allow_sql: blocks for execute-sql action\n \"\"\"\n processor = ConfigPermissionProcessor(datasette, actor, action)\n result = processor.process()\n\n if result is None:\n return []\n\n return [result]\n" }, { "path": "datasette/default_permissions/defaults.py", "content": "\"\"\"\nDefault permission settings for Datasette.\n\nProvides default allow rules for standard view/execute actions.\n\"\"\"\n\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING, Optional\n\nif TYPE_CHECKING:\n from datasette.app import Datasette\n\nfrom datasette import hookimpl\nfrom datasette.permissions import PermissionSQL\n\n# Actions that are allowed by default (unless --default-deny is used)\nDEFAULT_ALLOW_ACTIONS = frozenset(\n {\n \"view-instance\",\n \"view-database\",\n \"view-database-download\",\n \"view-table\",\n \"view-query\",\n \"execute-sql\",\n }\n)\n\n\n@hookimpl(specname=\"permission_resources_sql\")\nasync def default_allow_sql_check(\n datasette: \"Datasette\",\n actor: Optional[dict],\n action: str,\n) -> Optional[PermissionSQL]:\n \"\"\"\n Enforce the default_allow_sql setting.\n\n When default_allow_sql is false (the default), execute-sql is denied\n unless explicitly allowed by config or other rules.\n \"\"\"\n if action == \"execute-sql\":\n if not datasette.setting(\"default_allow_sql\"):\n return PermissionSQL.deny(reason=\"default_allow_sql is false\")\n\n return None\n\n\n@hookimpl(specname=\"permission_resources_sql\")\nasync def default_action_permissions_sql(\n datasette: \"Datasette\",\n actor: Optional[dict],\n action: str,\n) -> Optional[PermissionSQL]:\n \"\"\"\n Provide default allow rules for standard view/execute actions.\n\n These defaults are skipped when datasette is started with --default-deny.\n The restriction_sql mechanism (from actor_restrictions_sql) will still\n filter these results if the actor has restrictions.\n \"\"\"\n if datasette.default_deny:\n return None\n\n if action in DEFAULT_ALLOW_ACTIONS:\n reason = f\"default allow for {action}\".replace(\"'\", \"''\")\n return PermissionSQL.allow(reason=reason)\n\n return None\n" }, { "path": "datasette/default_permissions/helpers.py", "content": "\"\"\"\nShared helper utilities for default permission implementations.\n\"\"\"\n\nfrom __future__ import annotations\n\nfrom dataclasses import dataclass\nfrom typing import TYPE_CHECKING, List, Optional, Set\n\nif TYPE_CHECKING:\n from datasette.app import Datasette\n\nfrom datasette.permissions import PermissionSQL\n\n\ndef get_action_name_variants(datasette: \"Datasette\", action: str) -> Set[str]:\n \"\"\"\n Get all name variants for an action (full name and abbreviation).\n\n Example:\n get_action_name_variants(ds, \"view-table\") -> {\"view-table\", \"vt\"}\n \"\"\"\n variants = {action}\n action_obj = datasette.actions.get(action)\n if action_obj and action_obj.abbr:\n variants.add(action_obj.abbr)\n return variants\n\n\ndef action_in_list(datasette: \"Datasette\", action: str, action_list: list) -> bool:\n \"\"\"Check if an action (or its abbreviation) is in a list.\"\"\"\n return bool(get_action_name_variants(datasette, action).intersection(action_list))\n\n\n@dataclass\nclass PermissionRow:\n \"\"\"A single permission rule row.\"\"\"\n\n parent: Optional[str]\n child: Optional[str]\n allow: bool\n reason: str\n\n\nclass PermissionRowCollector:\n \"\"\"Collects permission rows and converts them to PermissionSQL.\"\"\"\n\n def __init__(self, prefix: str = \"row\"):\n self.rows: List[PermissionRow] = []\n self.prefix = prefix\n\n def add(\n self,\n parent: Optional[str],\n child: Optional[str],\n allow: Optional[bool],\n reason: str,\n if_not_none: bool = False,\n ) -> None:\n \"\"\"Add a permission row. If if_not_none=True, only add if allow is not None.\"\"\"\n if if_not_none and allow is None:\n return\n self.rows.append(PermissionRow(parent, child, allow, reason))\n\n def to_permission_sql(self) -> Optional[PermissionSQL]:\n \"\"\"Convert collected rows to a PermissionSQL object.\"\"\"\n if not self.rows:\n return None\n\n parts = []\n params = {}\n\n for idx, row in enumerate(self.rows):\n key = f\"{self.prefix}_{idx}\"\n parts.append(\n f\"SELECT :{key}_parent AS parent, :{key}_child AS child, \"\n f\":{key}_allow AS allow, :{key}_reason AS reason\"\n )\n params[f\"{key}_parent\"] = row.parent\n params[f\"{key}_child\"] = row.child\n params[f\"{key}_allow\"] = 1 if row.allow else 0\n params[f\"{key}_reason\"] = row.reason\n\n sql = \"\\nUNION ALL\\n\".join(parts)\n return PermissionSQL(sql=sql, params=params)\n" }, { "path": "datasette/default_permissions/restrictions.py", "content": "\"\"\"\nActor restriction handling for Datasette permissions.\n\nThis module handles the _r (restrictions) key in actor dictionaries, which\ncontains allowlists of resources the actor can access.\n\"\"\"\n\nfrom __future__ import annotations\n\nfrom dataclasses import dataclass\nfrom typing import TYPE_CHECKING, List, Optional, Set, Tuple\n\nif TYPE_CHECKING:\n from datasette.app import Datasette\n\nfrom datasette import hookimpl\nfrom datasette.permissions import PermissionSQL\n\nfrom .helpers import action_in_list, get_action_name_variants\n\n\n@dataclass\nclass ActorRestrictions:\n \"\"\"Parsed actor restrictions from the _r key.\"\"\"\n\n global_actions: List[str] # _r.a - globally allowed actions\n database_actions: dict # _r.d - {db_name: [actions]}\n table_actions: dict # _r.r - {db_name: {table: [actions]}}\n\n @classmethod\n def from_actor(cls, actor: Optional[dict]) -> Optional[\"ActorRestrictions\"]:\n \"\"\"Parse restrictions from actor dict. Returns None if no restrictions.\"\"\"\n if not actor:\n return None\n assert isinstance(actor, dict), \"actor must be a dictionary\"\n\n restrictions = actor.get(\"_r\")\n if restrictions is None:\n return None\n\n return cls(\n global_actions=restrictions.get(\"a\", []),\n database_actions=restrictions.get(\"d\", {}),\n table_actions=restrictions.get(\"r\", {}),\n )\n\n def is_action_globally_allowed(self, datasette: \"Datasette\", action: str) -> bool:\n \"\"\"Check if action is in the global allowlist.\"\"\"\n return action_in_list(datasette, action, self.global_actions)\n\n def get_allowed_databases(self, datasette: \"Datasette\", action: str) -> Set[str]:\n \"\"\"Get database names where this action is allowed.\"\"\"\n allowed = set()\n for db_name, db_actions in self.database_actions.items():\n if action_in_list(datasette, action, db_actions):\n allowed.add(db_name)\n return allowed\n\n def get_allowed_tables(\n self, datasette: \"Datasette\", action: str\n ) -> Set[Tuple[str, str]]:\n \"\"\"Get (database, table) pairs where this action is allowed.\"\"\"\n allowed = set()\n for db_name, tables in self.table_actions.items():\n for table_name, table_actions in tables.items():\n if action_in_list(datasette, action, table_actions):\n allowed.add((db_name, table_name))\n return allowed\n\n\n@hookimpl(specname=\"permission_resources_sql\")\nasync def actor_restrictions_sql(\n datasette: \"Datasette\",\n actor: Optional[dict],\n action: str,\n) -> Optional[List[PermissionSQL]]:\n \"\"\"\n Handle actor restriction-based permission rules.\n\n When an actor has an \"_r\" key, it contains an allowlist of resources they\n can access. This function returns restriction_sql that filters the final\n results to only include resources in that allowlist.\n\n The _r structure:\n {\n \"a\": [\"vi\", \"pd\"], # Global actions allowed\n \"d\": {\"mydb\": [\"vt\", \"es\"]}, # Database-level actions\n \"r\": {\"mydb\": {\"users\": [\"vt\"]}} # Table-level actions\n }\n \"\"\"\n if not actor:\n return None\n\n restrictions = ActorRestrictions.from_actor(actor)\n\n if restrictions is None:\n # No restrictions - all resources allowed\n return []\n\n # If globally allowed, no filtering needed\n if restrictions.is_action_globally_allowed(datasette, action):\n return []\n\n # Build restriction SQL\n allowed_dbs = restrictions.get_allowed_databases(datasette, action)\n allowed_tables = restrictions.get_allowed_tables(datasette, action)\n\n # If nothing is allowed for this action, return empty-set restriction\n if not allowed_dbs and not allowed_tables:\n return [\n PermissionSQL(\n params={\"deny\": f\"actor restrictions: {action} not in allowlist\"},\n restriction_sql=\"SELECT NULL AS parent, NULL AS child WHERE 0\",\n )\n ]\n\n # Build UNION of allowed resources\n selects = []\n params = {}\n counter = 0\n\n # Database-level entries (parent, NULL) - allows all children\n for db_name in allowed_dbs:\n key = f\"restr_{counter}\"\n counter += 1\n selects.append(f\"SELECT :{key}_parent AS parent, NULL AS child\")\n params[f\"{key}_parent\"] = db_name\n\n # Table-level entries (parent, child)\n for db_name, table_name in allowed_tables:\n key = f\"restr_{counter}\"\n counter += 1\n selects.append(f\"SELECT :{key}_parent AS parent, :{key}_child AS child\")\n params[f\"{key}_parent\"] = db_name\n params[f\"{key}_child\"] = table_name\n\n restriction_sql = \"\\nUNION ALL\\n\".join(selects)\n\n return [PermissionSQL(params=params, restriction_sql=restriction_sql)]\n\n\ndef restrictions_allow_action(\n datasette: \"Datasette\",\n restrictions: dict,\n action: str,\n resource: Optional[str | Tuple[str, str]],\n) -> bool:\n \"\"\"\n Check if restrictions allow the requested action on the requested resource.\n\n This is a synchronous utility function for use by other code that needs\n to quickly check restriction allowlists.\n\n Args:\n datasette: The Datasette instance\n restrictions: The _r dict from an actor\n action: The action name to check\n resource: None for global, str for database, (db, table) tuple for table\n\n Returns:\n True if allowed, False if denied\n \"\"\"\n # Does this action have an abbreviation?\n to_check = get_action_name_variants(datasette, action)\n\n # Check global level (any resource)\n all_allowed = restrictions.get(\"a\")\n if all_allowed is not None:\n assert isinstance(all_allowed, list)\n if to_check.intersection(all_allowed):\n return True\n\n # Check database level\n if resource:\n if isinstance(resource, str):\n database_name = resource\n else:\n database_name = resource[0]\n database_allowed = restrictions.get(\"d\", {}).get(database_name)\n if database_allowed is not None:\n assert isinstance(database_allowed, list)\n if to_check.intersection(database_allowed):\n return True\n\n # Check table/resource level\n if resource is not None and not isinstance(resource, str) and len(resource) == 2:\n database, table = resource\n table_allowed = restrictions.get(\"r\", {}).get(database, {}).get(table)\n if table_allowed is not None:\n assert isinstance(table_allowed, list)\n if to_check.intersection(table_allowed):\n return True\n\n # This action is not explicitly allowed, so reject it\n return False\n" }, { "path": "datasette/default_permissions/root.py", "content": "\"\"\"\nRoot user permission handling for Datasette.\n\nGrants full permissions to the root user when --root flag is used.\n\"\"\"\n\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING, Optional\n\nif TYPE_CHECKING:\n from datasette.app import Datasette\n\nfrom datasette import hookimpl\nfrom datasette.permissions import PermissionSQL\n\n\n@hookimpl(specname=\"permission_resources_sql\")\nasync def root_user_permissions_sql(\n datasette: \"Datasette\",\n actor: Optional[dict],\n) -> Optional[PermissionSQL]:\n \"\"\"\n Grant root user full permissions when --root flag is used.\n \"\"\"\n if not datasette.root_enabled:\n return None\n if actor is not None and actor.get(\"id\") == \"root\":\n return PermissionSQL.allow(reason=\"root user\")\n" }, { "path": "datasette/default_permissions/tokens.py", "content": "\"\"\"\nToken authentication for Datasette.\n\nRegisters the default SignedTokenHandler and delegates token verification\nto datasette.verify_token() so all registered handlers are tried.\n\"\"\"\n\nfrom __future__ import annotations\n\nfrom typing import TYPE_CHECKING, Optional\n\nif TYPE_CHECKING:\n from datasette.app import Datasette\n\nfrom datasette import hookimpl\nfrom datasette.tokens import SignedTokenHandler\n\n\n@hookimpl\ndef register_token_handler(datasette: \"Datasette\"):\n \"\"\"Register the default signed token handler.\"\"\"\n return SignedTokenHandler()\n\n\n@hookimpl(specname=\"actor_from_request\")\nasync def actor_from_signed_api_token(\n datasette: \"Datasette\", request\n) -> Optional[dict]:\n \"\"\"\n Authenticate requests using API tokens by delegating to all registered\n token handlers via datasette.verify_token().\n \"\"\"\n authorization = request.headers.get(\"authorization\")\n if not authorization:\n return None\n if not authorization.startswith(\"Bearer \"):\n return None\n\n token = authorization[len(\"Bearer \") :]\n return await datasette.verify_token(token)\n" }, { "path": "datasette/events.py", "content": "from abc import ABC, abstractproperty\nfrom dataclasses import asdict, dataclass, field\nfrom datasette.hookspecs import hookimpl\nfrom datetime import datetime, timezone\n\n\n@dataclass\nclass Event(ABC):\n @abstractproperty\n def name(self):\n pass\n\n created: datetime = field(\n init=False, default_factory=lambda: datetime.now(timezone.utc)\n )\n actor: dict | None\n\n def properties(self):\n properties = asdict(self)\n properties.pop(\"actor\", None)\n properties.pop(\"created\", None)\n return properties\n\n\n@dataclass\nclass LoginEvent(Event):\n \"\"\"\n Event name: ``login``\n\n A user (represented by ``event.actor``) has logged in.\n \"\"\"\n\n name = \"login\"\n\n\n@dataclass\nclass LogoutEvent(Event):\n \"\"\"\n Event name: ``logout``\n\n A user (represented by ``event.actor``) has logged out.\n \"\"\"\n\n name = \"logout\"\n\n\n@dataclass\nclass CreateTokenEvent(Event):\n \"\"\"\n Event name: ``create-token``\n\n A user created an API token.\n\n :ivar expires_after: Number of seconds after which this token will expire.\n :type expires_after: int or None\n :ivar restrict_all: Restricted permissions for this token.\n :type restrict_all: list\n :ivar restrict_database: Restricted database permissions for this token.\n :type restrict_database: dict\n :ivar restrict_resource: Restricted resource permissions for this token.\n :type restrict_resource: dict\n \"\"\"\n\n name = \"create-token\"\n expires_after: int | None\n restrict_all: list\n restrict_database: dict\n restrict_resource: dict\n\n\n@dataclass\nclass CreateTableEvent(Event):\n \"\"\"\n Event name: ``create-table``\n\n A new table has been created in the database.\n\n :ivar database: The name of the database where the table was created.\n :type database: str\n :ivar table: The name of the table that was created\n :type table: str\n :ivar schema: The SQL schema definition for the new table.\n :type schema: str\n \"\"\"\n\n name = \"create-table\"\n database: str\n table: str\n schema: str\n\n\n@dataclass\nclass DropTableEvent(Event):\n \"\"\"\n Event name: ``drop-table``\n\n A table has been dropped from the database.\n\n :ivar database: The name of the database where the table was dropped.\n :type database: str\n :ivar table: The name of the table that was dropped\n :type table: str\n \"\"\"\n\n name = \"drop-table\"\n database: str\n table: str\n\n\n@dataclass\nclass AlterTableEvent(Event):\n \"\"\"\n Event name: ``alter-table``\n\n A table has been altered.\n\n :ivar database: The name of the database where the table was altered\n :type database: str\n :ivar table: The name of the table that was altered\n :type table: str\n :ivar before_schema: The table's SQL schema before the alteration\n :type before_schema: str\n :ivar after_schema: The table's SQL schema after the alteration\n :type after_schema: str\n \"\"\"\n\n name = \"alter-table\"\n database: str\n table: str\n before_schema: str\n after_schema: str\n\n\n@dataclass\nclass InsertRowsEvent(Event):\n \"\"\"\n Event name: ``insert-rows``\n\n Rows were inserted into a table.\n\n :ivar database: The name of the database where the rows were inserted.\n :type database: str\n :ivar table: The name of the table where the rows were inserted.\n :type table: str\n :ivar num_rows: The number of rows that were requested to be inserted.\n :type num_rows: int\n :ivar ignore: Was ignore set?\n :type ignore: bool\n :ivar replace: Was replace set?\n :type replace: bool\n \"\"\"\n\n name = \"insert-rows\"\n database: str\n table: str\n num_rows: int\n ignore: bool\n replace: bool\n\n\n@dataclass\nclass UpsertRowsEvent(Event):\n \"\"\"\n Event name: ``upsert-rows``\n\n Rows were upserted into a table.\n\n :ivar database: The name of the database where the rows were inserted.\n :type database: str\n :ivar table: The name of the table where the rows were inserted.\n :type table: str\n :ivar num_rows: The number of rows that were requested to be inserted.\n :type num_rows: int\n \"\"\"\n\n name = \"upsert-rows\"\n database: str\n table: str\n num_rows: int\n\n\n@dataclass\nclass UpdateRowEvent(Event):\n \"\"\"\n Event name: ``update-row``\n\n A row was updated in a table.\n\n :ivar database: The name of the database where the row was updated.\n :type database: str\n :ivar table: The name of the table where the row was updated.\n :type table: str\n :ivar pks: The primary key values of the updated row.\n \"\"\"\n\n name = \"update-row\"\n database: str\n table: str\n pks: list\n\n\n@dataclass\nclass DeleteRowEvent(Event):\n \"\"\"\n Event name: ``delete-row``\n\n A row was deleted from a table.\n\n :ivar database: The name of the database where the row was deleted.\n :type database: str\n :ivar table: The name of the table where the row was deleted.\n :type table: str\n :ivar pks: The primary key values of the deleted row.\n \"\"\"\n\n name = \"delete-row\"\n database: str\n table: str\n pks: list\n\n\n@hookimpl\ndef register_events():\n return [\n LoginEvent,\n LogoutEvent,\n CreateTableEvent,\n CreateTokenEvent,\n AlterTableEvent,\n DropTableEvent,\n InsertRowsEvent,\n UpsertRowsEvent,\n UpdateRowEvent,\n DeleteRowEvent,\n ]\n" }, { "path": "datasette/facets.py", "content": "import json\nimport urllib\nfrom datasette import hookimpl\nfrom datasette.database import QueryInterrupted\nfrom datasette.utils import (\n escape_sqlite,\n path_with_added_args,\n path_with_removed_args,\n detect_json1,\n sqlite3,\n)\n\n\ndef load_facet_configs(request, table_config):\n # Given a request and the configuration for a table, return\n # a dictionary of selected facets, their lists of configs and for each\n # config whether it came from the request or the metadata.\n #\n # return {type: [\n # {\"source\": \"metadata\", \"config\": config1},\n # {\"source\": \"request\", \"config\": config2}]}\n facet_configs = {}\n table_config = table_config or {}\n table_facet_configs = table_config.get(\"facets\", [])\n for facet_config in table_facet_configs:\n if isinstance(facet_config, str):\n type = \"column\"\n facet_config = {\"simple\": facet_config}\n else:\n assert (\n len(facet_config.values()) == 1\n ), \"Metadata config dicts should be {type: config}\"\n type, facet_config = list(facet_config.items())[0]\n if isinstance(facet_config, str):\n facet_config = {\"simple\": facet_config}\n facet_configs.setdefault(type, []).append(\n {\"source\": \"metadata\", \"config\": facet_config}\n )\n qs_pairs = urllib.parse.parse_qs(request.query_string, keep_blank_values=True)\n for key, values in qs_pairs.items():\n if key.startswith(\"_facet\"):\n # Figure out the facet type\n if key == \"_facet\":\n type = \"column\"\n elif key.startswith(\"_facet_\"):\n type = key[len(\"_facet_\") :]\n for value in values:\n # The value is the facet_config - either JSON or not\n facet_config = (\n json.loads(value) if value.startswith(\"{\") else {\"simple\": value}\n )\n facet_configs.setdefault(type, []).append(\n {\"source\": \"request\", \"config\": facet_config}\n )\n return facet_configs\n\n\n@hookimpl\ndef register_facet_classes():\n classes = [ColumnFacet, DateFacet]\n if detect_json1():\n classes.append(ArrayFacet)\n return classes\n\n\nclass Facet:\n type = None\n # How many rows to consider when suggesting facets:\n suggest_consider = 1000\n\n def __init__(\n self,\n ds,\n request,\n database,\n sql=None,\n table=None,\n params=None,\n table_config=None,\n row_count=None,\n ):\n assert table or sql, \"Must provide either table= or sql=\"\n self.ds = ds\n self.request = request\n self.database = database\n # For foreign key expansion. Can be None for e.g. canned SQL queries:\n self.table = table\n self.sql = sql or f\"select * from [{table}]\"\n self.params = params or []\n self.table_config = table_config\n # row_count can be None, in which case we calculate it ourselves:\n self.row_count = row_count\n\n def get_configs(self):\n configs = load_facet_configs(self.request, self.table_config)\n return configs.get(self.type) or []\n\n def get_querystring_pairs(self):\n # ?_foo=bar&_foo=2&empty= becomes:\n # [('_foo', 'bar'), ('_foo', '2'), ('empty', '')]\n return urllib.parse.parse_qsl(self.request.query_string, keep_blank_values=True)\n\n def get_facet_size(self):\n facet_size = self.ds.setting(\"default_facet_size\")\n max_returned_rows = self.ds.setting(\"max_returned_rows\")\n table_facet_size = None\n if self.table:\n config_facet_size = (\n self.ds.config.get(\"databases\", {})\n .get(self.database, {})\n .get(\"tables\", {})\n .get(self.table, {})\n .get(\"facet_size\")\n )\n if config_facet_size:\n table_facet_size = config_facet_size\n custom_facet_size = self.request.args.get(\"_facet_size\")\n if custom_facet_size:\n if custom_facet_size == \"max\":\n facet_size = max_returned_rows\n elif custom_facet_size.isdigit():\n facet_size = int(custom_facet_size)\n else:\n # Invalid value, ignore it\n custom_facet_size = None\n if table_facet_size and not custom_facet_size:\n if table_facet_size == \"max\":\n facet_size = max_returned_rows\n else:\n facet_size = table_facet_size\n return min(facet_size, max_returned_rows)\n\n async def suggest(self):\n return []\n\n async def facet_results(self):\n # returns ([results], [timed_out])\n # TODO: Include \"hideable\" with each one somehow, which indicates if it was\n # defined in metadata (in which case you cannot turn it off)\n raise NotImplementedError\n\n async def get_columns(self, sql, params=None):\n # Detect column names using the \"limit 0\" trick\n return (\n await self.ds.execute(\n self.database, f\"select * from ({sql}) limit 0\", params or []\n )\n ).columns\n\n\nclass ColumnFacet(Facet):\n type = \"column\"\n\n async def suggest(self):\n row_count = await self.get_row_count()\n columns = await self.get_columns(self.sql, self.params)\n facet_size = self.get_facet_size()\n suggested_facets = []\n already_enabled = [c[\"config\"][\"simple\"] for c in self.get_configs()]\n for column in columns:\n if column in already_enabled:\n continue\n suggested_facet_sql = \"\"\"\n with limited as (select * from ({sql}) limit {suggest_consider})\n select {column} as value, count(*) as n from limited\n where value is not null\n group by value\n limit {limit}\n \"\"\".format(\n column=escape_sqlite(column),\n sql=self.sql,\n limit=facet_size + 1,\n suggest_consider=self.suggest_consider,\n )\n distinct_values = None\n try:\n distinct_values = await self.ds.execute(\n self.database,\n suggested_facet_sql,\n self.params,\n truncate=False,\n custom_time_limit=self.ds.setting(\"facet_suggest_time_limit_ms\"),\n )\n num_distinct_values = len(distinct_values)\n if (\n 1 < num_distinct_values < row_count\n and num_distinct_values <= facet_size\n # And at least one has n > 1\n and any(r[\"n\"] > 1 for r in distinct_values)\n ):\n suggested_facets.append(\n {\n \"name\": column,\n \"toggle_url\": self.ds.absolute_url(\n self.request,\n self.ds.urls.path(\n path_with_added_args(\n self.request, {\"_facet\": column}\n )\n ),\n ),\n }\n )\n except QueryInterrupted:\n continue\n return suggested_facets\n\n async def get_row_count(self):\n if self.row_count is None:\n self.row_count = (\n await self.ds.execute(\n self.database,\n f\"select count(*) from (select * from ({self.sql}) limit {self.suggest_consider})\",\n self.params,\n )\n ).rows[0][0]\n return self.row_count\n\n async def facet_results(self):\n facet_results = []\n facets_timed_out = []\n\n qs_pairs = self.get_querystring_pairs()\n\n facet_size = self.get_facet_size()\n for source_and_config in self.get_configs():\n config = source_and_config[\"config\"]\n source = source_and_config[\"source\"]\n column = config.get(\"column\") or config[\"simple\"]\n facet_sql = \"\"\"\n select {col} as value, count(*) as count from (\n {sql}\n )\n where {col} is not null\n group by {col} order by count desc, value limit {limit}\n \"\"\".format(col=escape_sqlite(column), sql=self.sql, limit=facet_size + 1)\n try:\n facet_rows_results = await self.ds.execute(\n self.database,\n facet_sql,\n self.params,\n truncate=False,\n custom_time_limit=self.ds.setting(\"facet_time_limit_ms\"),\n )\n facet_results_values = []\n facet_results.append(\n {\n \"name\": column,\n \"type\": self.type,\n \"hideable\": source != \"metadata\",\n \"toggle_url\": self.ds.urls.path(\n path_with_removed_args(self.request, {\"_facet\": column})\n ),\n \"results\": facet_results_values,\n \"truncated\": len(facet_rows_results) > facet_size,\n }\n )\n facet_rows = facet_rows_results.rows[:facet_size]\n if self.table:\n # Attempt to expand foreign keys into labels\n values = [row[\"value\"] for row in facet_rows]\n expanded = await self.ds.expand_foreign_keys(\n self.request.actor, self.database, self.table, column, values\n )\n else:\n expanded = {}\n for row in facet_rows:\n column_qs = column\n if column.startswith(\"_\"):\n column_qs = \"{}__exact\".format(column)\n selected = (column_qs, str(row[\"value\"])) in qs_pairs\n if selected:\n toggle_path = path_with_removed_args(\n self.request, {column_qs: str(row[\"value\"])}\n )\n else:\n toggle_path = path_with_added_args(\n self.request, {column_qs: row[\"value\"]}\n )\n facet_results_values.append(\n {\n \"value\": row[\"value\"],\n \"label\": expanded.get((column, row[\"value\"]), row[\"value\"]),\n \"count\": row[\"count\"],\n \"toggle_url\": self.ds.absolute_url(\n self.request, self.ds.urls.path(toggle_path)\n ),\n \"selected\": selected,\n }\n )\n except QueryInterrupted:\n facets_timed_out.append(column)\n\n return facet_results, facets_timed_out\n\n\nclass ArrayFacet(Facet):\n type = \"array\"\n\n def _is_json_array_of_strings(self, json_string):\n try:\n array = json.loads(json_string)\n except ValueError:\n return False\n for item in array:\n if not isinstance(item, str):\n return False\n return True\n\n async def suggest(self):\n columns = await self.get_columns(self.sql, self.params)\n suggested_facets = []\n already_enabled = [c[\"config\"][\"simple\"] for c in self.get_configs()]\n for column in columns:\n if column in already_enabled:\n continue\n # Is every value in this column either null or a JSON array?\n suggested_facet_sql = \"\"\"\n with limited as (select * from ({sql}) limit {suggest_consider})\n select distinct json_type({column})\n from limited\n where {column} is not null and {column} != ''\n \"\"\".format(\n column=escape_sqlite(column),\n sql=self.sql,\n suggest_consider=self.suggest_consider,\n )\n try:\n results = await self.ds.execute(\n self.database,\n suggested_facet_sql,\n self.params,\n truncate=False,\n custom_time_limit=self.ds.setting(\"facet_suggest_time_limit_ms\"),\n log_sql_errors=False,\n )\n types = tuple(r[0] for r in results.rows)\n if types in ((\"array\",), (\"array\", None)):\n # Now check that first 100 arrays contain only strings\n first_100 = [\n v[0]\n for v in await self.ds.execute(\n self.database,\n (\n \"select {column} from ({sql}) \"\n \"where {column} is not null \"\n \"and {column} != '' \"\n \"and json_array_length({column}) > 0 \"\n \"limit 100\"\n ).format(column=escape_sqlite(column), sql=self.sql),\n self.params,\n truncate=False,\n custom_time_limit=self.ds.setting(\n \"facet_suggest_time_limit_ms\"\n ),\n log_sql_errors=False,\n )\n ]\n if first_100 and all(\n self._is_json_array_of_strings(r) for r in first_100\n ):\n suggested_facets.append(\n {\n \"name\": column,\n \"type\": \"array\",\n \"toggle_url\": self.ds.absolute_url(\n self.request,\n self.ds.urls.path(\n path_with_added_args(\n self.request, {\"_facet_array\": column}\n )\n ),\n ),\n }\n )\n except (QueryInterrupted, sqlite3.OperationalError):\n continue\n return suggested_facets\n\n async def facet_results(self):\n # self.configs should be a plain list of columns\n facet_results = []\n facets_timed_out = []\n\n facet_size = self.get_facet_size()\n for source_and_config in self.get_configs():\n config = source_and_config[\"config\"]\n source = source_and_config[\"source\"]\n column = config.get(\"column\") or config[\"simple\"]\n # https://github.com/simonw/datasette/issues/448\n facet_sql = \"\"\"\n with inner as ({sql}),\n deduped_array_items as (\n select\n distinct j.value,\n inner.*\n from\n json_each([inner].{col}) j\n join inner\n )\n select\n value as value,\n count(*) as count\n from\n deduped_array_items\n group by\n value\n order by\n count(*) desc, value limit {limit}\n \"\"\".format(\n col=escape_sqlite(column),\n sql=self.sql,\n limit=facet_size + 1,\n )\n try:\n facet_rows_results = await self.ds.execute(\n self.database,\n facet_sql,\n self.params,\n truncate=False,\n custom_time_limit=self.ds.setting(\"facet_time_limit_ms\"),\n )\n facet_results_values = []\n facet_results.append(\n {\n \"name\": column,\n \"type\": self.type,\n \"results\": facet_results_values,\n \"hideable\": source != \"metadata\",\n \"toggle_url\": self.ds.urls.path(\n path_with_removed_args(\n self.request, {\"_facet_array\": column}\n )\n ),\n \"truncated\": len(facet_rows_results) > facet_size,\n }\n )\n facet_rows = facet_rows_results.rows[:facet_size]\n pairs = self.get_querystring_pairs()\n for row in facet_rows:\n value = str(row[\"value\"])\n selected = (f\"{column}__arraycontains\", value) in pairs\n if selected:\n toggle_path = path_with_removed_args(\n self.request, {f\"{column}__arraycontains\": value}\n )\n else:\n toggle_path = path_with_added_args(\n self.request, {f\"{column}__arraycontains\": value}\n )\n facet_results_values.append(\n {\n \"value\": value,\n \"label\": value,\n \"count\": row[\"count\"],\n \"toggle_url\": self.ds.absolute_url(\n self.request, toggle_path\n ),\n \"selected\": selected,\n }\n )\n except QueryInterrupted:\n facets_timed_out.append(column)\n\n return facet_results, facets_timed_out\n\n\nclass DateFacet(Facet):\n type = \"date\"\n\n async def suggest(self):\n columns = await self.get_columns(self.sql, self.params)\n already_enabled = [c[\"config\"][\"simple\"] for c in self.get_configs()]\n suggested_facets = []\n for column in columns:\n if column in already_enabled:\n continue\n # Does this column contain any dates in the first 100 rows?\n suggested_facet_sql = \"\"\"\n select date({column}) from (\n select * from ({sql}) limit 100\n ) where {column} glob \"????-??-*\"\n \"\"\".format(column=escape_sqlite(column), sql=self.sql)\n try:\n results = await self.ds.execute(\n self.database,\n suggested_facet_sql,\n self.params,\n truncate=False,\n custom_time_limit=self.ds.setting(\"facet_suggest_time_limit_ms\"),\n log_sql_errors=False,\n )\n values = tuple(r[0] for r in results.rows)\n if any(values):\n suggested_facets.append(\n {\n \"name\": column,\n \"type\": \"date\",\n \"toggle_url\": self.ds.absolute_url(\n self.request,\n self.ds.urls.path(\n path_with_added_args(\n self.request, {\"_facet_date\": column}\n )\n ),\n ),\n }\n )\n except (QueryInterrupted, sqlite3.OperationalError):\n continue\n return suggested_facets\n\n async def facet_results(self):\n facet_results = []\n facets_timed_out = []\n args = dict(self.get_querystring_pairs())\n facet_size = self.get_facet_size()\n for source_and_config in self.get_configs():\n config = source_and_config[\"config\"]\n source = source_and_config[\"source\"]\n column = config.get(\"column\") or config[\"simple\"]\n # TODO: does this query break if inner sql produces value or count columns?\n facet_sql = \"\"\"\n select date({col}) as value, count(*) as count from (\n {sql}\n )\n where date({col}) is not null\n group by date({col}) order by count desc, value limit {limit}\n \"\"\".format(col=escape_sqlite(column), sql=self.sql, limit=facet_size + 1)\n try:\n facet_rows_results = await self.ds.execute(\n self.database,\n facet_sql,\n self.params,\n truncate=False,\n custom_time_limit=self.ds.setting(\"facet_time_limit_ms\"),\n )\n facet_results_values = []\n facet_results.append(\n {\n \"name\": column,\n \"type\": self.type,\n \"results\": facet_results_values,\n \"hideable\": source != \"metadata\",\n \"toggle_url\": path_with_removed_args(\n self.request, {\"_facet_date\": column}\n ),\n \"truncated\": len(facet_rows_results) > facet_size,\n }\n )\n facet_rows = facet_rows_results.rows[:facet_size]\n for row in facet_rows:\n selected = str(args.get(f\"{column}__date\")) == str(row[\"value\"])\n if selected:\n toggle_path = path_with_removed_args(\n self.request, {f\"{column}__date\": str(row[\"value\"])}\n )\n else:\n toggle_path = path_with_added_args(\n self.request, {f\"{column}__date\": row[\"value\"]}\n )\n facet_results_values.append(\n {\n \"value\": row[\"value\"],\n \"label\": row[\"value\"],\n \"count\": row[\"count\"],\n \"toggle_url\": self.ds.absolute_url(\n self.request, toggle_path\n ),\n \"selected\": selected,\n }\n )\n except QueryInterrupted:\n facets_timed_out.append(column)\n\n return facet_results, facets_timed_out\n" }, { "path": "datasette/filters.py", "content": "from datasette import hookimpl\nfrom datasette.resources import DatabaseResource\nfrom datasette.views.base import DatasetteError\nfrom datasette.utils.asgi import BadRequest\nimport json\nfrom .utils import detect_json1, escape_sqlite, path_with_removed_args\n\n\n@hookimpl(specname=\"filters_from_request\")\ndef where_filters(request, database, datasette):\n # This one deals with ?_where=\n async def inner():\n where_clauses = []\n extra_wheres_for_ui = []\n if \"_where\" in request.args:\n if not await datasette.allowed(\n action=\"execute-sql\",\n resource=DatabaseResource(database=database),\n actor=request.actor,\n ):\n raise DatasetteError(\"_where= is not allowed\", status=403)\n else:\n where_clauses.extend(request.args.getlist(\"_where\"))\n extra_wheres_for_ui = [\n {\n \"text\": text,\n \"remove_url\": path_with_removed_args(request, {\"_where\": text}),\n }\n for text in request.args.getlist(\"_where\")\n ]\n\n return FilterArguments(\n where_clauses,\n extra_context={\n \"extra_wheres_for_ui\": extra_wheres_for_ui,\n },\n )\n\n return inner\n\n\n@hookimpl(specname=\"filters_from_request\")\ndef search_filters(request, database, table, datasette):\n # ?_search= and _search_colname=\n async def inner():\n where_clauses = []\n params = {}\n human_descriptions = []\n extra_context = {}\n\n # Figure out which fts_table to use\n table_metadata = await datasette.table_config(database, table)\n db = datasette.get_database(database)\n fts_table = request.args.get(\"_fts_table\")\n fts_table = fts_table or table_metadata.get(\"fts_table\")\n fts_table = fts_table or await db.fts_table(table)\n fts_pk = request.args.get(\"_fts_pk\", table_metadata.get(\"fts_pk\", \"rowid\"))\n search_args = {\n key: request.args[key]\n for key in request.args\n if key.startswith(\"_search\") and key != \"_searchmode\"\n }\n search = \"\"\n search_mode_raw = table_metadata.get(\"searchmode\") == \"raw\"\n # Or set search mode from the querystring\n qs_searchmode = request.args.get(\"_searchmode\")\n if qs_searchmode == \"escaped\":\n search_mode_raw = False\n if qs_searchmode == \"raw\":\n search_mode_raw = True\n\n extra_context[\"supports_search\"] = bool(fts_table)\n\n if fts_table and search_args:\n if \"_search\" in search_args:\n # Simple ?_search=xxx\n search = search_args[\"_search\"]\n where_clauses.append(\n \"{fts_pk} in (select rowid from {fts_table} where {fts_table} match {match_clause})\".format(\n fts_table=escape_sqlite(fts_table),\n fts_pk=escape_sqlite(fts_pk),\n match_clause=(\n \":search\" if search_mode_raw else \"escape_fts(:search)\"\n ),\n )\n )\n human_descriptions.append(f'search matches \"{search}\"')\n params[\"search\"] = search\n extra_context[\"search\"] = search\n else:\n # More complex: search against specific columns\n for i, (key, search_text) in enumerate(search_args.items()):\n search_col = key.split(\"_search_\", 1)[1]\n if search_col not in await db.table_columns(fts_table):\n raise BadRequest(\"Cannot search by that column\")\n\n where_clauses.append(\n \"rowid in (select rowid from {fts_table} where {search_col} match {match_clause})\".format(\n fts_table=escape_sqlite(fts_table),\n search_col=escape_sqlite(search_col),\n match_clause=(\n \":search_{}\".format(i)\n if search_mode_raw\n else \"escape_fts(:search_{})\".format(i)\n ),\n )\n )\n human_descriptions.append(\n f'search column \"{search_col}\" matches \"{search_text}\"'\n )\n params[f\"search_{i}\"] = search_text\n extra_context[\"search\"] = search_text\n\n return FilterArguments(where_clauses, params, human_descriptions, extra_context)\n\n return inner\n\n\n@hookimpl(specname=\"filters_from_request\")\ndef through_filters(request, database, table, datasette):\n # ?_search= and _search_colname=\n async def inner():\n where_clauses = []\n params = {}\n human_descriptions = []\n extra_context = {}\n\n # Support for ?_through={table, column, value}\n if \"_through\" in request.args:\n for through in request.args.getlist(\"_through\"):\n through_data = json.loads(through)\n through_table = through_data[\"table\"]\n other_column = through_data[\"column\"]\n value = through_data[\"value\"]\n db = datasette.get_database(database)\n outgoing_foreign_keys = await db.foreign_keys_for_table(through_table)\n try:\n fk_to_us = [\n fk for fk in outgoing_foreign_keys if fk[\"other_table\"] == table\n ][0]\n except IndexError:\n raise DatasetteError(\n \"Invalid _through - could not find corresponding foreign key\"\n )\n param = f\"p{len(params)}\"\n where_clauses.append(\n \"{our_pk} in (select {our_column} from {through_table} where {other_column} = :{param})\".format(\n through_table=escape_sqlite(through_table),\n our_pk=escape_sqlite(fk_to_us[\"other_column\"]),\n our_column=escape_sqlite(fk_to_us[\"column\"]),\n other_column=escape_sqlite(other_column),\n param=param,\n )\n )\n params[param] = value\n human_descriptions.append(f'{through_table}.{other_column} = \"{value}\"')\n\n return FilterArguments(where_clauses, params, human_descriptions, extra_context)\n\n return inner\n\n\nclass FilterArguments:\n def __init__(\n self, where_clauses, params=None, human_descriptions=None, extra_context=None\n ):\n self.where_clauses = where_clauses\n self.params = params or {}\n self.human_descriptions = human_descriptions or []\n self.extra_context = extra_context or {}\n\n\nclass Filter:\n key = None\n display = None\n no_argument = False\n\n def where_clause(self, table, column, value, param_counter):\n raise NotImplementedError\n\n def human_clause(self, column, value):\n raise NotImplementedError\n\n\nclass TemplatedFilter(Filter):\n def __init__(\n self,\n key,\n display,\n sql_template,\n human_template,\n format=\"{}\",\n numeric=False,\n no_argument=False,\n ):\n self.key = key\n self.display = display\n self.sql_template = sql_template\n self.human_template = human_template\n self.format = format\n self.numeric = numeric\n self.no_argument = no_argument\n\n def where_clause(self, table, column, value, param_counter):\n converted = self.format.format(value)\n if self.numeric and converted.isdigit():\n converted = int(converted)\n if self.no_argument:\n kwargs = {\"c\": column}\n converted = None\n else:\n kwargs = {\"c\": column, \"p\": f\"p{param_counter}\", \"t\": table}\n return self.sql_template.format(**kwargs), converted\n\n def human_clause(self, column, value):\n if callable(self.human_template):\n template = self.human_template(column, value)\n else:\n template = self.human_template\n if self.no_argument:\n return template.format(c=column)\n else:\n return template.format(c=column, v=value)\n\n\nclass InFilter(Filter):\n key = \"in\"\n display = \"in\"\n\n def split_value(self, value):\n if value.startswith(\"[\"):\n return json.loads(value)\n else:\n return [v.strip() for v in value.split(\",\")]\n\n def where_clause(self, table, column, value, param_counter):\n values = self.split_value(value)\n params = [f\":p{param_counter + i}\" for i in range(len(values))]\n sql = f\"{escape_sqlite(column)} in ({', '.join(params)})\"\n return sql, values\n\n def human_clause(self, column, value):\n return f\"{column} in {json.dumps(self.split_value(value))}\"\n\n\nclass NotInFilter(InFilter):\n key = \"notin\"\n display = \"not in\"\n\n def where_clause(self, table, column, value, param_counter):\n values = self.split_value(value)\n params = [f\":p{param_counter + i}\" for i in range(len(values))]\n sql = f\"{escape_sqlite(column)} not in ({', '.join(params)})\"\n return sql, values\n\n def human_clause(self, column, value):\n return f\"{column} not in {json.dumps(self.split_value(value))}\"\n\n\nclass Filters:\n _filters = (\n [\n # key, display, sql_template, human_template, format=, numeric=, no_argument=\n TemplatedFilter(\n \"exact\",\n \"=\",\n '\"{c}\" = :{p}',\n lambda c, v: \"{c} = {v}\" if v.isdigit() else '{c} = \"{v}\"',\n ),\n TemplatedFilter(\n \"not\",\n \"!=\",\n '\"{c}\" != :{p}',\n lambda c, v: \"{c} != {v}\" if v.isdigit() else '{c} != \"{v}\"',\n ),\n TemplatedFilter(\n \"contains\",\n \"contains\",\n '\"{c}\" like :{p}',\n '{c} contains \"{v}\"',\n format=\"%{}%\",\n ),\n TemplatedFilter(\n \"notcontains\",\n \"does not contain\",\n '\"{c}\" not like :{p}',\n '{c} does not contain \"{v}\"',\n format=\"%{}%\",\n ),\n TemplatedFilter(\n \"endswith\",\n \"ends with\",\n '\"{c}\" like :{p}',\n '{c} ends with \"{v}\"',\n format=\"%{}\",\n ),\n TemplatedFilter(\n \"startswith\",\n \"starts with\",\n '\"{c}\" like :{p}',\n '{c} starts with \"{v}\"',\n format=\"{}%\",\n ),\n TemplatedFilter(\"gt\", \">\", '\"{c}\" > :{p}', \"{c} > {v}\", numeric=True),\n TemplatedFilter(\n \"gte\", \"\\u2265\", '\"{c}\" >= :{p}', \"{c} \\u2265 {v}\", numeric=True\n ),\n TemplatedFilter(\"lt\", \"<\", '\"{c}\" < :{p}', \"{c} < {v}\", numeric=True),\n TemplatedFilter(\n \"lte\", \"\\u2264\", '\"{c}\" <= :{p}', \"{c} \\u2264 {v}\", numeric=True\n ),\n TemplatedFilter(\"like\", \"like\", '\"{c}\" like :{p}', '{c} like \"{v}\"'),\n TemplatedFilter(\n \"notlike\", \"not like\", '\"{c}\" not like :{p}', '{c} not like \"{v}\"'\n ),\n TemplatedFilter(\"glob\", \"glob\", '\"{c}\" glob :{p}', '{c} glob \"{v}\"'),\n InFilter(),\n NotInFilter(),\n ]\n + (\n [\n TemplatedFilter(\n \"arraycontains\",\n \"array contains\",\n \"\"\":{p} in (select value from json_each([{t}].[{c}]))\"\"\",\n '{c} contains \"{v}\"',\n ),\n TemplatedFilter(\n \"arraynotcontains\",\n \"array does not contain\",\n \"\"\":{p} not in (select value from json_each([{t}].[{c}]))\"\"\",\n '{c} does not contain \"{v}\"',\n ),\n ]\n if detect_json1()\n else []\n )\n + [\n TemplatedFilter(\n \"date\", \"date\", 'date(\"{c}\") = :{p}', '\"{c}\" is on date {v}'\n ),\n TemplatedFilter(\n \"isnull\", \"is null\", '\"{c}\" is null', \"{c} is null\", no_argument=True\n ),\n TemplatedFilter(\n \"notnull\",\n \"is not null\",\n '\"{c}\" is not null',\n \"{c} is not null\",\n no_argument=True,\n ),\n TemplatedFilter(\n \"isblank\",\n \"is blank\",\n '(\"{c}\" is null or \"{c}\" = \"\")',\n \"{c} is blank\",\n no_argument=True,\n ),\n TemplatedFilter(\n \"notblank\",\n \"is not blank\",\n '(\"{c}\" is not null and \"{c}\" != \"\")',\n \"{c} is not blank\",\n no_argument=True,\n ),\n ]\n )\n _filters_by_key = {f.key: f for f in _filters}\n\n def __init__(self, pairs):\n self.pairs = pairs\n\n def lookups(self):\n \"\"\"Yields (lookup, display, no_argument) pairs\"\"\"\n for filter in self._filters:\n yield filter.key, filter.display, filter.no_argument\n\n def human_description_en(self, extra=None):\n bits = []\n if extra:\n bits.extend(extra)\n for column, lookup, value in self.selections():\n filter = self._filters_by_key.get(lookup, None)\n if filter:\n bits.append(filter.human_clause(column, value))\n # Comma separated, with an ' and ' at the end\n and_bits = []\n commas, tail = bits[:-1], bits[-1:]\n if commas:\n and_bits.append(\", \".join(commas))\n if tail:\n and_bits.append(tail[0])\n s = \" and \".join(and_bits)\n if not s:\n return \"\"\n return f\"where {s}\"\n\n def selections(self):\n \"\"\"Yields (column, lookup, value) tuples\"\"\"\n for key, value in self.pairs:\n if \"__\" in key:\n column, lookup = key.rsplit(\"__\", 1)\n else:\n column = key\n lookup = \"exact\"\n yield column, lookup, value\n\n def has_selections(self):\n return bool(self.pairs)\n\n def build_where_clauses(self, table):\n sql_bits = []\n params = {}\n i = 0\n for column, lookup, value in self.selections():\n filter = self._filters_by_key.get(lookup, None)\n if filter:\n sql_bit, param = filter.where_clause(table, column, value, i)\n sql_bits.append(sql_bit)\n if param is not None:\n if not isinstance(param, list):\n param = [param]\n for individual_param in param:\n param_id = f\"p{i}\"\n params[param_id] = individual_param\n i += 1\n return sql_bits, params\n" }, { "path": "datasette/forbidden.py", "content": "from datasette import hookimpl, Response\n\n\n@hookimpl(trylast=True)\ndef forbidden(datasette, request, message):\n async def inner():\n return Response.html(\n await datasette.render_template(\n \"error.html\",\n {\n \"title\": \"Forbidden\",\n \"error\": message,\n },\n request=request,\n ),\n status=403,\n )\n\n return inner\n" }, { "path": "datasette/handle_exception.py", "content": "from datasette import hookimpl, Response\nfrom .utils import add_cors_headers\nfrom .utils.asgi import (\n Base400,\n)\nfrom .views.base import DatasetteError\nfrom markupsafe import Markup\nimport traceback\n\ntry:\n import ipdb as pdb\nexcept ImportError:\n import pdb\n\ntry:\n import rich\nexcept ImportError:\n rich = None\n\n\n@hookimpl(trylast=True)\ndef handle_exception(datasette, request, exception):\n async def inner():\n if datasette.pdb:\n pdb.post_mortem(exception.__traceback__)\n\n if rich is not None:\n rich.get_console().print_exception(show_locals=True)\n\n title = None\n if isinstance(exception, Base400):\n status = exception.status\n info = {}\n message = exception.args[0]\n elif isinstance(exception, DatasetteError):\n status = exception.status\n info = exception.error_dict\n message = exception.message\n if exception.message_is_html:\n message = Markup(message)\n title = exception.title\n else:\n status = 500\n info = {}\n message = str(exception)\n traceback.print_exc()\n templates = [f\"{status}.html\", \"error.html\"]\n info.update(\n {\n \"ok\": False,\n \"error\": message,\n \"status\": status,\n \"title\": title,\n }\n )\n headers = {}\n if datasette.cors:\n add_cors_headers(headers)\n if request.path.split(\"?\")[0].endswith(\".json\"):\n return Response.json(info, status=status, headers=headers)\n else:\n environment = datasette.get_jinja_environment(request)\n template = environment.select_template(templates)\n return Response.html(\n await template.render_async(\n dict(\n info,\n urls=datasette.urls,\n app_css_hash=datasette.app_css_hash(),\n menu_links=lambda: [],\n )\n ),\n status=status,\n headers=headers,\n )\n\n return inner\n" }, { "path": "datasette/hookspecs.py", "content": "from pluggy import HookimplMarker\nfrom pluggy import HookspecMarker\n\nhookspec = HookspecMarker(\"datasette\")\nhookimpl = HookimplMarker(\"datasette\")\n\n\n@hookspec\ndef startup(datasette):\n \"\"\"Fires directly after Datasette first starts running\"\"\"\n\n\n@hookspec\ndef asgi_wrapper(datasette):\n \"\"\"Returns an ASGI middleware callable to wrap our ASGI application with\"\"\"\n\n\n@hookspec\ndef prepare_connection(conn, database, datasette):\n \"\"\"Modify SQLite connection in some way e.g. register custom SQL functions\"\"\"\n\n\n@hookspec\ndef prepare_jinja2_environment(env, datasette):\n \"\"\"Modify Jinja2 template environment e.g. register custom template tags\"\"\"\n\n\n@hookspec\ndef extra_css_urls(template, database, table, columns, view_name, request, datasette):\n \"\"\"Extra CSS URLs added by this plugin\"\"\"\n\n\n@hookspec\ndef extra_js_urls(template, database, table, columns, view_name, request, datasette):\n \"\"\"Extra JavaScript URLs added by this plugin\"\"\"\n\n\n@hookspec\ndef extra_body_script(\n template, database, table, columns, view_name, request, datasette\n):\n \"\"\"Extra JavaScript code to be included in \n" }, { "path": "datasette/templates/_codemirror.html", "content": "\n\n\n" }, { "path": "datasette/templates/_codemirror_foot.html", "content": "\n" }, { "path": "datasette/templates/_crumbs.html", "content": "{% macro nav(request, database=None, table=None) -%}\n{% if crumb_items is defined %}\n {% set items=crumb_items(request=request, database=database, table=table) %}\n {% if items %}\n

\n {% for item in items %}\n {{ item.label }}\n {% if not loop.last %}\n /\n {% endif %}\n {% endfor %}\n

\n {% endif %}\n{% endif %}\n{%- endmacro %}\n" }, { "path": "datasette/templates/_debug_common_functions.html", "content": "\n" }, { "path": "datasette/templates/_description_source_license.html", "content": "{% if metadata.get(\"description_html\") or metadata.get(\"description\") %}\n
\n {% if metadata.get(\"description_html\") %}\n {{ metadata.description_html|safe }}\n {% else %}\n {{ metadata.description }}\n {% endif %}\n
\n{% endif %}\n{% if metadata.license or metadata.license_url or metadata.source or metadata.source_url %}\n

\n {% if metadata.license or metadata.license_url %}Data license:\n {% if metadata.license_url %}\n {{ metadata.license or metadata.license_url }}\n {% else %}\n {{ metadata.license }}\n {% endif %}\n {% endif %}\n {% if metadata.source or metadata.source_url %}{% if metadata.license or metadata.license_url %}·{% endif %}\n Data source: {% if metadata.source_url %}\n \n {% endif %}{{ metadata.source or metadata.source_url }}{% if metadata.source_url %}{% endif %}\n {% endif %}\n {% if metadata.about or metadata.about_url %}{% if metadata.license or metadata.license_url or metadata.source or metadata.source_url %}·{% endif %}\n About: {% if metadata.about_url %}\n \n {% endif %}{{ metadata.about or metadata.about_url }}{% if metadata.about_url %}{% endif %}\n {% endif %}\n

\n{% endif %}\n" }, { "path": "datasette/templates/_facet_results.html", "content": "
\n {% for facet_info in sorted_facet_results %}\n
\n

\n {{ facet_info.name }}{% if facet_info.type != \"column\" %} ({{ facet_info.type }}){% endif %}\n {% if facet_info.truncated %}>{% endif %}{{ facet_info.results|length }}\n \n {% if facet_info.hideable %}\n ✖\n {% endif %}\n

\n
    \n {% for facet_value in facet_info.results %}\n {% if not facet_value.selected %}\n
  • {{ (facet_value.label | string()) or \"-\" }} {{ \"{:,}\".format(facet_value.count) }}
    • \n {% else %}\n
  • {{ facet_value.label or \"-\" }} · {{ \"{:,}\".format(facet_value.count) }} ✖
    • \n {% endif %}\n {% endfor %}\n {% if facet_info.truncated %}\n
  • {% if request.args._facet_size != \"max\" -%}\n …{% else -%}…{% endif %}\n
    • \n {% endif %}\n
\n
\n {% endfor %}\n
\n" }, { "path": "datasette/templates/_footer.html", "content": "Powered by Datasette\n{% if query_ms %}· Queries took {{ query_ms|round(3) }}ms{% endif %}\n{% if metadata %}\n {% if metadata.license or metadata.license_url %}· Data license:\n {% if metadata.license_url %}\n {{ metadata.license or metadata.license_url }}\n {% else %}\n {{ metadata.license }}\n {% endif %}\n {% endif %}\n {% if metadata.source or metadata.source_url %}·\n Data source: {% if metadata.source_url %}\n \n {% endif %}{{ metadata.source or metadata.source_url }}{% if metadata.source_url %}{% endif %}\n {% endif %}\n {% if metadata.about or metadata.about_url %}·\n About: {% if metadata.about_url %}\n \n {% endif %}{{ metadata.about or metadata.about_url }}{% if metadata.about_url %}{% endif %}\n {% endif %}\n{% endif %}\n" }, { "path": "datasette/templates/_permission_ui_styles.html", "content": "\n" }, { "path": "datasette/templates/_permissions_debug_tabs.html", "content": "{% if has_debug_permission %}\n{% set query_string = '?' + request.query_string if request.query_string else '' %}\n\n\n\n\n{% endif %}\n" }, { "path": "datasette/templates/_suggested_facets.html", "content": "

\n Suggested facets: {% for facet in suggested_facets %}{{ facet.name }}{% if facet.get(\"type\") %} ({{ facet.type }}){% endif %}{% if not loop.last %}, {% endif %}{% endfor %}\n

\n" }, { "path": "datasette/templates/_table.html", "content": "\n
\n{% if display_rows %}\n
\n
\n \n \n {% for column in display_columns %}\n \n {% endfor %}\n \n \n \n {% for row in display_rows %}\n \n {% for cell in row %}\n \n {% endfor %}\n \n {% endfor %}\n \n
\n {% if not column.sortable %}\n {{ column.name }}\n {% else %}\n {% if column.name == sort %}\n {{ column.name }} ▼\n {% else %}\n {{ column.name }}{% if column.name == sort_desc %} ▲{% endif %}\n {% endif %}\n {% endif %}\n
{{ cell.value }}
\n\n{% else %}\n

0 records

\n{% endif %}\n" }, { "path": "datasette/templates/allow_debug.html", "content": "{% extends \"base.html\" %}\n\n{% block title %}Debug allow rules{% endblock %}\n\n{% block extra_head %}\n\n{% endblock %}\n\n{% block content %}\n\n

Debug allow rules

\n\n{% set current_tab = \"allow_debug\" %}\n{% include \"_permissions_debug_tabs.html\" %}\n\n

Use this tool to try out different actor and allow combinations. See Defining permissions with \"allow\" blocks for documentation.

\n\n
\n
\n

\n \n
\n
\n

\n \n
\n
\n \n
\n
\n\n{% if error %}

{{ error }}

{% endif %}\n\n{% if result == \"True\" %}

Result: allow

{% endif %}\n\n{% if result == \"False\" %}

Result: deny

{% endif %}\n\n{% endblock %}\n" }, { "path": "datasette/templates/api_explorer.html", "content": "{% extends \"base.html\" %}\n\n{% block title %}API Explorer{% endblock %}\n\n{% block extra_head %}\n\n{% endblock %}\n\n{% block content %}\n\n

API Explorer{% if private %} 🔒{% endif %}

\n\n

Use this tool to try out the\n {% if datasette_version %}\n Datasette API.\n {% else %}\n Datasette API.\n {% endif %}\n

\n
\n GET\n
\n
\n \n \n \n
\n
\n
\n
\n POST\n
\n
\n \n \n
\n
\n \n \n
\n

\n
\n
\n\n
\n

API response: HTTP

\n \n
    \n 
    \n
    \n\n\n\n{% if example_links %}\n
    API endpoints
    \n
      \n  {% for database in example_links %}\n    
    • Database: {{ database.name }}
      • \n    
        \n      {% for link in database.links %}\n        
      • {{ link.path }} - {{ link.label }} 
        • \n      {% endfor %}\n      {% for table in database.tables %}\n        
      • {{ table.name }}\n          
          \n            {% for link in table.links %}\n              
        • {{ link.path }} - {{ link.label }} 
          • \n            {% endfor %}\n          
        \n        
        • \n      {% endfor %}\n    
      \n  {% endfor %}\n
    \n{% endif %}\n\n{% endblock %}\n"
  },
  {
    "path": "datasette/templates/base.html",
    "content": "{% import \"_crumbs.html\" as crumbs with context %}\n\n\n    {% block title %}{% endblock %}\n    \n    \n{% for url in extra_css_urls %}\n    \n{% endfor %}\n\n\n{% for url in extra_js_urls %}\n    \n{% endfor %}\n{%- if alternate_url_json -%}\n    \n{%- endif -%}\n{%- block extra_head %}{% endblock -%}\n\n\n
    \n
    \n\n{% block messages %}\n{% if show_messages %}\n    {% for message, message_type in show_messages() %}\n        
    {{ message }}
    \n    {% endfor %}\n{% endif %}\n{% endblock %}\n\n
    \n{% block content %}\n{% endblock %}\n
    \n
    \n
    {% block footer %}{% include \"_footer.html\" %}{% endblock %}
    \n\n{% include \"_close_open_menus.html\" %}\n\n{% for body_script in body_scripts %}\n    {{ body_script.script }}\n{% endfor %}\n\n{% if select_templates %}{% endif %}\n\n\n\n\n"
  },
  {
    "path": "datasette/templates/create_token.html",
    "content": "{% extends \"base.html\" %}\n\n{% block title %}Create an API token{% endblock %}\n\n{% block extra_head %}\n\n{% endblock %}\n\n{% block content %}\n\n
    Create an API token
    \n\n
    This token will allow API access with the same abilities as your current user, {{ request.actor.id }}
    \n\n{% if token %}\n  
    \n    
    Your API token
    \n    
    \n      \n      \n    
    \n    \n    
    \n      Token details\n      
    {{ token_bits|tojson(4) }}
    \n    
    \n  
    \n  
    Create another token
    \n{% endif %}\n\n{% if errors %}\n  {% for error in errors %}\n    
    {{ error }}
    \n  {% endfor %}\n{% endif %}\n\n
    \n  
    \n    
    \n      \n    
    \n    \n    \n    \n\n  
    \n    Restrict actions that can be performed using this token\n    
    All databases and tables
    \n    
      \n      {% for permission in all_actions %}\n        
      • \n      {% endfor %}\n    
    \n\n    {% for database in database_with_tables %}\n      
    All tables in \"{{ database.name }}\"
    \n      
      \n        {% for permission in database_actions %}\n          
      • \n        {% endfor %}\n      
    \n    {% endfor %}\n    
    Specific tables
    \n    {% for database in database_with_tables %}\n      {% for table in database.tables %}\n        
    {{ database.name }}: {{ table.name }}
    \n        
      \n          {% for permission in child_actions %}\n            
      • \n          {% endfor %}\n        
    \n      {% endfor %}\n    {% endfor %}\n  
    \n\n\n
    \n\n\n\n{% endblock %}\n"
  },
  {
    "path": "datasette/templates/csrf_error.html",
    "content": "{% extends \"base.html\" %}\n{% block title %}CSRF check failed){% endblock %}\n{% block content %}\n
    Form origin check failed
    \n\n
    Your request's origin could not be validated. Please return to the form and submit it again.
    \n\n
    Technical details\n  
    Developers: consult Datasette's CSRF protection documentation.
    \n  
    Error code is {{ message_name }}.
    \n
    \n\n{% endblock %}\n"
  },
  {
    "path": "datasette/templates/database.html",
    "content": "{% extends \"base.html\" %}\n\n{% block title %}{{ database }}{% endblock %}\n\n{% block extra_head %}\n{{- super() -}}\n{% include \"_codemirror.html\" %}\n{% endblock %}\n\n{% block body_class %}db db-{{ database|to_css_class }}{% endblock %}\n\n{% block crumbs %}\n{{ crumbs.nav(request=request, database=database) }}\n{% endblock %}\n\n{% block content %}\n
    \n    
    {{ metadata.title or database }}{% if private %} 🔒{% endif %}
    \n
    \n{% set action_links, action_title = database_actions(), \"Database actions\" %}\n{% include \"_action_menu.html\" %}\n\n{{ top_database() }}\n\n{% block description_source_license %}{% include \"_description_source_license.html\" %}{% endblock %}\n\n{% if allow_execute_sql %}\n    
    \n        
    Custom SQL query
    \n        
    \n        
    \n            \n            \n        
    \n    
    \n{% endif %}\n\n{% if attached_databases %}\n    
    \n        
    The following databases are attached to this connection, and can be used for cross-database joins:
    \n        
      \n            {% for db_name in attached_databases %}\n                
    • {{ db_name }} - tables
      • \n            {% endfor %}\n        
    \n    
    \n{% endif %}\n\n{% if queries %}\n    
    Queries
    \n    \n{% endif %}\n\n{% if tables %}\n
    Tables schema
    \n{% endif %}\n\n{% for table in tables %}\n{% if show_hidden or not table.hidden %}\n
    \n    
    {{ table.name }}{% if table.private %} 🔒{% endif %}{% if table.hidden %} (hidden){% endif %}
    \n    
    {% for column in table.columns %}{{ column }}{% if not loop.last %}, {% endif %}{% endfor %}
    \n    
    {% if table.count is none %}Many rows{% elif table.count == count_limit + 1 %}>{{ \"{:,}\".format(count_limit) }} rows{% else %}{{ \"{:,}\".format(table.count) }} row{% if table.count == 1 %}{% else %}s{% endif %}{% endif %}
    \n
    \n{% endif %}\n{% endfor %}\n\n{% if hidden_count and not show_hidden %}\n    
    ... and {{ \"{:,}\".format(hidden_count) }} hidden table{% if hidden_count == 1 %}{% else %}s{% endif %}
    \n{% endif %}\n\n{% if views %}\n    
    Views
    \n    
      \n        {% for view in views %}\n            
    • {{ view.name }}{% if view.private %} 🔒{% endif %}
      • \n        {% endfor %}\n    
    \n{% endif %}\n\n{% if allow_download %}\n    
    Download SQLite DB: {{ database }}.db {{ format_bytes(size) }}
    \n{% endif %}\n\n{% include \"_codemirror_foot.html\" %}\n\n{% endblock %}\n"
  },
  {
    "path": "datasette/templates/debug_actions.html",
    "content": "{% extends \"base.html\" %}\n\n{% block title %}Registered Actions{% endblock %}\n\n{% block content %}\n
    Registered actions
    \n\n{% set current_tab = \"actions\" %}\n{% include \"_permissions_debug_tabs.html\" %}\n\n
    \n  This Datasette instance has registered {{ data|length }} action{{ data|length != 1 and \"s\" or \"\" }}.\n  Actions are used by the permission system to control access to different features.\n
    \n\n\n  \n    \n      \n      \n      \n      \n      \n      \n      \n    \n  \n  \n    {% for action in data %}\n    \n      \n      \n      \n      \n      \n      \n      \n    \n    {% endfor %}\n  \n
    NameAbbrDescriptionResourceTakes ParentTakes ChildAlso Requires
    {{ action.name }}{% if action.abbr %}{{ action.abbr }}{% endif %}{{ action.description or \"\" }}{% if action.resource_class %}{{ action.resource_class }}{% endif %}{% if action.takes_parent %}✓{% endif %}{% if action.takes_child %}✓{% endif %}{% if action.also_requires %}{{ action.also_requires }}{% endif %}
    \n\n{% endblock %}\n"
  },
  {
    "path": "datasette/templates/debug_allowed.html",
    "content": "{% extends \"base.html\" %}\n\n{% block title %}Allowed Resources{% endblock %}\n\n{% block extra_head %}\n\n{% include \"_permission_ui_styles.html\" %}\n{% include \"_debug_common_functions.html\" %}\n{% endblock %}\n\n{% block content %}\n
    Allowed resources
    \n\n{% set current_tab = \"allowed\" %}\n{% include \"_permissions_debug_tabs.html\" %}\n\n
    Use this tool to check which resources the current actor is allowed to access for a given permission action. It queries the /-/allowed.json API endpoint.
    \n\n{% if request.actor %}\n
    Current actor: {{ request.actor.get(\"id\", \"anonymous\") }}
    \n{% else %}\n
    Current actor: anonymous (not logged in)
    \n{% endif %}\n\n
    \n    
    \n        
    \n            \n            \n            Only certain actions are supported by this endpoint\n        
    \n\n        
    \n            \n            \n            Filter results to a specific parent resource\n        
    \n\n        
    \n            \n            \n            Filter results to a specific child resource (requires parent to be set)\n        
    \n\n        
    \n            \n            \n            Number of results per page (max 200)\n        
    \n\n        
    \n            \n        
    \n    
    \n
    \n\n
    \n    
    \n        
    Results
    \n        
    \n    
    \n\n    
    \n\n    
    \n\n    
    \n        Raw JSON response\n        
    \n    
    \n
    \n\n\n\n{% endblock %}\n"
  },
  {
    "path": "datasette/templates/debug_check.html",
    "content": "{% extends \"base.html\" %}\n\n{% block title %}Permission Check{% endblock %}\n\n{% block extra_head %}\n\n{% include \"_permission_ui_styles.html\" %}\n{% include \"_debug_common_functions.html\" %}\n\n{% endblock %}\n\n{% block content %}\n
    Permission check
    \n\n{% set current_tab = \"check\" %}\n{% include \"_permissions_debug_tabs.html\" %}\n\n
    Use this tool to test permission checks for the current actor. It queries the /-/check.json API endpoint.
    \n\n{% if request.actor %}\n
    Current actor: {{ request.actor.get(\"id\", \"anonymous\") }}
    \n{% else %}\n
    Current actor: anonymous (not logged in)
    \n{% endif %}\n\n
    \n    
    \n        
    \n            \n            \n            The permission action to check\n        
    \n\n        
    \n            \n            \n            For database-level permissions, specify the database name\n        
    \n\n        
    \n            \n            \n            For table-level permissions, specify the table name (requires parent)\n        
    \n\n        
    \n            \n        
    \n    
    \n
    \n\n
    \n    
    Result: 
    \n\n    
    \n        
    Action:
    \n        
    \n\n        
    Resource Path:
    \n        
    \n\n        
    Actor ID:
    \n        
    \n\n        
    \n    
    \n\n    
    \n        Raw JSON response\n        
    \n    
    \n
    \n\n\n\n{% endblock %}\n"
  },
  {
    "path": "datasette/templates/debug_permissions_playground.html",
    "content": "{% extends \"base.html\" %}\n\n{% block title %}Debug permissions{% endblock %}\n\n{% block extra_head %}\n{% include \"_permission_ui_styles.html\" %}\n\n{% endblock %}\n\n{% block content %}\n
    Permission playground
    \n\n{% set current_tab = \"permissions\" %}\n{% include \"_permissions_debug_tabs.html\" %}\n\n
    This tool lets you simulate an actor and a permission check for that actor.
    \n\n
    \n    
    \n        \n        
    \n            
    \n                \n                \n            
    \n        
    \n        
    \n            
    \n                \n                \n            
    \n            
    \n                \n                \n            
    \n            
    \n                \n                \n            
    \n        
    \n        
    \n            \n        
    \n        
    \n    
    \n
    \n\n\n\n
    Recent permissions checks
    \n\n
    \n    {% if filter != \"all\" %}All{% else %}All{% endif %},\n    {% if filter != \"exclude-yours\" %}Exclude yours{% else %}Exclude yours{% endif %},\n    {% if filter != \"only-yours\" %}Only yours{% else %}Only yours{% endif %}\n
    \n\n{% if permission_checks %}\n\n    \n        \n            \n            \n            \n            \n            \n            \n        \n    \n    \n        {% for check in permission_checks %}\n            \n                \n                \n                \n                \n                \n                \n            \n        {% endfor %}\n        \n
    WhenActionParentChildActorResult
    {{ check.when.split('T', 1)[0] }}
    {{ check.when.split('T', 1)[1].split('+', 1)[0].split('-', 1)[0].split('Z', 1)[0] }}    		{{ check.action }}{{ check.parent or '—' }}{{ check.child or '—' }}{% if check.actor %}{{ check.actor|tojson }}{% else %}anonymous{% endif %}{% if check.result %}Allowed{% elif check.result is none %}No opinion{% else %}Denied{% endif %}
    \n{% else %}\n
    No permission checks have been recorded yet.
    \n{% endif %}\n\n{% endblock %}\n"
  },
  {
    "path": "datasette/templates/debug_rules.html",
    "content": "{% extends \"base.html\" %}\n\n{% block title %}Permission Rules{% endblock %}\n\n{% block extra_head %}\n\n{% include \"_permission_ui_styles.html\" %}\n{% include \"_debug_common_functions.html\" %}\n{% endblock %}\n\n{% block content %}\n
    Permission rules
    \n\n{% set current_tab = \"rules\" %}\n{% include \"_permissions_debug_tabs.html\" %}\n\n
    Use this tool to view the permission rules that allow the current actor to access resources for a given permission action. It queries the /-/rules.json API endpoint.
    \n\n{% if request.actor %}\n
    Current actor: {{ request.actor.get(\"id\", \"anonymous\") }}
    \n{% else %}\n
    Current actor: anonymous (not logged in)
    \n{% endif %}\n\n
    \n    
    \n        
    \n            \n            \n            The permission action to check\n        
    \n\n        
    \n            \n            \n            Number of results per page (max 200)\n        
    \n\n        
    \n            \n        
    \n    
    \n
    \n\n
    \n    
    \n        
    Results
    \n        
    \n    
    \n\n    
    \n\n    
    \n\n    
    \n        Raw JSON response\n        
    \n    
    \n
    \n\n\n\n{% endblock %}\n"
  },
  {
    "path": "datasette/templates/error.html",
    "content": "{% extends \"base.html\" %}\n\n{% block title %}{% if title %}{{ title }}{% else %}Error {{ status }}{% endif %}{% endblock %}\n\n{% block content %}\n\n
    {% if title %}{{ title }}{% else %}Error {{ status }}{% endif %}
    \n\n
    {{ error }}
    \n\n{% endblock %}\n"
  },
  {
    "path": "datasette/templates/index.html",
    "content": "{% extends \"base.html\" %}\n\n{% block title %}{{ metadata.title or \"Datasette\" }}: {% for database in databases %}{{ database.name }}{% if not loop.last %}, {% endif %}{% endfor %}{% endblock %}\n\n{% block extra_head %}\n{% if noindex %}{% endif %}\n{% endblock %}\n\n{% block body_class %}index{% endblock %}\n\n{% block content %}\n
    {{ metadata.title or \"Datasette\" }}{% if private %} 🔒{% endif %}
    \n\n{% set action_links, action_title = homepage_actions, \"Homepage actions\" %}\n{% include \"_action_menu.html\" %}\n\n{{ top_homepage() }}\n\n{% block description_source_license %}{% include \"_description_source_license.html\" %}{% endblock %}\n\n{% for database in databases %}\n    
    {{ database.name }}{% if database.private %} 🔒{% endif %}
    \n    
    \n        {% if database.show_table_row_counts %}{{ \"{:,}\".format(database.table_rows_sum) }} rows in {% endif %}{{ database.tables_count }} table{% if database.tables_count != 1 %}s{% endif %}{% if database.hidden_tables_count %}, {% endif -%}\n        {% if database.hidden_tables_count -%}\n            {% if database.show_table_row_counts %}{{ \"{:,}\".format(database.hidden_table_rows_sum) }} rows in {% endif %}{{ database.hidden_tables_count }} hidden table{% if database.hidden_tables_count != 1 %}s{% endif -%}\n        {% endif -%}\n        {% if database.views_count -%}\n            {% if database.tables_count or database.hidden_tables_count %}, {% endif -%}\n            {{ \"{:,}\".format(database.views_count) }} view{% if database.views_count != 1 %}s{% endif %}\n        {% endif %}\n    
    \n    
    {% for table in database.tables_and_views_truncated %}{{ table.name }}{% if table.private %} 🔒{% endif %}{% if not loop.last %}, {% endif %}{% endfor %}{% if database.tables_and_views_more %}, ...{% endif %}
    \n{% endfor %}\n\n{% endblock %}\n"
  },
  {
    "path": "datasette/templates/logout.html",
    "content": "{% extends \"base.html\" %}\n\n{% block title %}Log out{% endblock %}\n\n{% block content %}\n\n
    Log out
    \n\n
    You are logged in as {{ display_actor(actor) }}
    \n\n
    \n    
    \n        \n        \n    
    \n
    \n\n{% endblock %}\n"
  },
  {
    "path": "datasette/templates/messages_debug.html",
    "content": "{% extends \"base.html\" %}\n\n{% block title %}Debug messages{% endblock %}\n\n{% block content %}\n\n
    Debug messages
    \n\n
    Set a message:
    \n\n
    \n    
    \n        \n        
    \n            \n        
    \n        \n        \n    
    \n
    \n\n{% endblock %}\n"
  },
  {
    "path": "datasette/templates/patterns.html",
    "content": "\n\n\n    Datasette: Pattern Portfolio\n    \n    \n    \n    \n\n\n\n
    \n\n
    \n    
    Pattern Portfolio
    \n
    \n\n\n\n\n
    Header for /database/table/row and Messages
    \n\n
    \n  \n
    \n\n
    Example message
    \n
    Example message
    \n
    Example message
    \n\n
    .bd for /
    \n
    \n    
    Datasette Fixtures
    \n    
    \n            An example SQLite database demonstrating Datasette\n    
    \n    
    \n    Data license:\n            Apache License 2.0\n    ·\n        Data source:\n            \n        tests/fixtures.py\n    ·\n        About:\n            \n        About Datasette\n    
    \n    
    fixtures
    \n    
    \n        1,258 rows in 24 tables, 206 rows in 5 hidden tables, 4 views\n    
    \n    
    compound_three_primary_keys, sortable, facetable, roadside_attraction_characteristics, simple_primary_key, ...
    \n    
    data
    \n    
    \n        6 rows in 2 tables\n    
    \n    
    names, foo
    \n
    \n\n
    .bd for /database
    \n
    \n    
    \n        
    fixtures
    \n    
    \n    
    \n        
    \n            \n                
    \n                    \n                        Database actions\n                        \n                        \n                    \n                    Database actions\n                
    \n                \n            
    \n                
    \n                \n            
    \n        
    \n    
    \n\n    
    \n            Test tables description\n    
    \n    
    \n    Data license:\n            Apache License 2.0\n    ·\n        Data source:\n            \n        tests/fixtures.py\n    ·\n        About:\n            \n        About Datasette\n    
    \n    
    \n        
    Custom SQL query
    \n        
    \n        
    \n            \n            \n        
    \n    
    \n    
    \n        
    123_starts_with_digits
    \n        
    content
    \n        
    0 rows
    \n    
    \n    
    \n        
    Table With Space In Name
    \n        
    pk, content
    \n        
    0 rows
    \n    
    \n    
    \n        
    attraction_characteristic
    \n        
    pk, name
    \n        
    2 rows
    \n    
    \n
    \n\n
    .bd for /database/table
    \n\n
    \n    
    \n        
    roadside_attraction_characteristics
    \n    
    \n    
    \n        
    \n            \n                
    \n                    \n                        Database actions\n                        \n                        \n                    \n                    Table actions\n                
    \n                \n            
    \n                
    \n                \n            
    \n        
    \n    
    \n\n    
    \n    Data license:\n            Apache License 2.0\n    ·\n        Data source:\n            \n        tests/fixtures.py\n    ·\n        About:\n            \n        About Datasette\n    
    \n    
    3 rows\n        where characteristic_id = 2\n    
    \n    
    \n    
    \n        
    \n            
    \n                \n            
    \n            
    \n                \n            
    \n        
    \n    
    \n        
    \n            \n        
    \n        
    \n            \n        
    \n    
    \n    
    \n            
    \n                \n            
    \n            \n        \n    
    \n  
    \n\n  
    \n    
    2 extra where clauses
    \n    \n  
    \n\n  
    ✎ View and edit SQL
    \n\n  
    This data as json, CSV (advanced)
    \n\n  
    \n        Suggested facets: tags, created (date), tags (array)\n    
    \n\n    
    \n\n            
    \n                
    \n                    tags (array)\n\n                        ✖\n\n                
    \n                \n            
    \n\n            
    \n                
    \n                    created\n\n                        ✖\n\n                
    \n                \n            
    \n\n            
    \n                
    \n                    city_id\n\n                        ✖\n\n                
    \n                \n            
    \n\n    
    \n\n    \n        \n            \n                    \n                    \n                    \n                    \n            \n        \n        \n            \n                    \n                    \n                    \n                    \n            \n            \n                    \n                    \n                    \n                    \n            \n            \n                    \n                    \n                    \n                    \n            \n        \n    
    \n                                Link\n                        \n                                            rowid ▼\n                                \n                                            attraction_id\n                                \n                                            characteristic_id\n                                
    11The Mystery Spot 1Paranormal 2
    22Winchester Mystery House 2Paranormal 2
    33Bigfoot Discovery Museum 4Paranormal 2
    \n    
    \n        
    Advanced export
    \n        
    JSON shape:\n            default,\n            array,\n            newline-delimited\n        
    \n        
    \n            
    \n                CSV options:\n                \n                \n                \n                \n                \n            
    \n        
    \n    
    \n    
    CREATE TABLE roadside_attraction_characteristics (\n    attraction_id INTEGER REFERENCES roadside_attractions(pk),\n    characteristic_id INTEGER REFERENCES attraction_characteristic(pk)\n);
    \n
    \n\n
    .bd for /database/table/row
    \n
    \n    
    roadside_attractions: 2
    \n    
    This data as json
    \n    \n        \n            \n                    \n                    \n                    \n                    \n                    \n            \n        \n        \n            \n                    \n                    \n                    \n                    \n                    \n            \n        \n    
    \n                                pk\n                        \n                                name\n                        \n                                address\n                        \n                                latitude\n                        \n                                longitude\n                        
    2Winchester Mystery House525 South Winchester Boulevard, San Jose, CA 9512837.3184-121.9511
    \n    
    Links from other tables
    \n    
      \n            
    • \n                \n                    1 row\n                from attraction_id in roadside_attraction_characteristics\n            
      • \n    
    \n
    \n\n
    .ft
    \n\n\n\n{% include \"_close_open_menus.html\" %}\n\n\n\n"
  },
  {
    "path": "datasette/templates/query.html",
    "content": "{% extends \"base.html\" %}\n\n{% block title %}{{ database }}{% if query and query.sql %}: {{ query.sql }}{% endif %}{% endblock %}\n\n{% block extra_head %}\n{{- super() -}}\n{% if columns %}\n\n{% endif %}\n{% include \"_codemirror.html\" %}\n{% endblock %}\n\n{% block body_class %}query db-{{ database|to_css_class }}{% if canned_query %} query-{{ canned_query|to_css_class }}{% endif %}{% endblock %}\n\n{% block crumbs %}\n{{ crumbs.nav(request=request, database=database) }}\n{% endblock %}\n\n{% block content %}\n\n{% if canned_query_write and db_is_immutable %}\n    
    This query cannot be executed because the database is immutable.
    \n{% endif %}\n\n
    {{ metadata.title or database }}{% if canned_query and not metadata.title %}: {{ canned_query }}{% endif %}{% if private %} 🔒{% endif %}
    \n{% set action_links, action_title = query_actions(), \"Query actions\" %}\n{% include \"_action_menu.html\" %}\n\n{% if canned_query %}{{ top_canned_query() }}{% else %}{{ top_query() }}{% endif %}\n\n{% block description_source_license %}{% include \"_description_source_license.html\" %}{% endblock %}\n\n
    \n    
    Custom SQL query{% if display_rows %} returning {% if truncated %}more than {% endif %}{{ \"{:,}\".format(display_rows|length) }} row{% if display_rows|length == 1 %}{% else %}s{% endif %}{% endif %}{% if not query_error %}\n        ({{ show_hide_text }})\n    {% endif %}
    \n    {% if error %}\n        
    {{ error }}
    \n    {% endif %}\n    {% if not hide_sql %}\n        {% if editable and allow_execute_sql %}\n            
    \n        {% else %}\n            
    {% if query %}{{ query.sql }}{% endif %}
    \n        {% endif %}\n    {% else %}\n        {% if not canned_query %}\n            \n        {% endif %}\n    {% endif %}\n    {% if named_parameter_values %}\n        
    Query parameters
    \n        {% for name, value in named_parameter_values.items() %}\n            
     
    \n        {% endfor %}\n    {% endif %}\n    
    \n        {% if not hide_sql %}{% endif %}\n        {% if canned_query_write %}{% endif %}\n        \n        {{ show_hide_hidden }}\n        {% if canned_query and edit_sql_url %}Edit SQL{% endif %}\n    
    \n
    \n\n{% if display_rows %}\n
    This data as {% for name, url in renderers.items() %}{{ name }}{{ \", \" if not loop.last }}{% endfor %}, CSV
    \n
    \n    \n        \n            {% for column in columns %}{% endfor %}\n        \n    \n    \n    {% for row in display_rows %}\n        \n            {% for column, td in zip(columns, row) %}\n                \n            {% endfor %}\n        \n    {% endfor %}\n    \n
    {{ column }}
    {{ td }}
    \n{% else %}\n    {% if not canned_query_write and not error %}\n        
    0 results
    \n    {% endif %}\n{% endif %}\n\n{% include \"_codemirror_foot.html\" %}\n\n{% endblock %}\n"
  },
  {
    "path": "datasette/templates/row.html",
    "content": "{% extends \"base.html\" %}\n\n{% block title %}{{ database }}: {{ table }}{% endblock %}\n\n{% block extra_head %}\n{{- super() -}}\n\n{% endblock %}\n\n{% block body_class %}row db-{{ database|to_css_class }} table-{{ table|to_css_class }}{% endblock %}\n\n{% block crumbs %}\n{{ crumbs.nav(request=request, database=database, table=table) }}\n{% endblock %}\n\n{% block content %}\n
    {{ table }}: {{ ', '.join(primary_key_values) }}{% if private %} 🔒{% endif %}
    \n\n{% set action_links, action_title = row_actions, \"Row actions\" %}\n{% include \"_action_menu.html\" %}\n\n{{ top_row() }}\n\n{% block description_source_license %}{% include \"_description_source_license.html\" %}{% endblock %}\n\n
    This data as {% for name, url in renderers.items() %}{{ name }}{{ \", \" if not loop.last }}{% endfor %}
    \n\n{% include custom_table_templates %}\n\n{% if foreign_key_tables %}\n    
    Links from other tables
    \n    \n{% endif %}\n\n{% endblock %}\n"
  },
  {
    "path": "datasette/templates/schema.html",
    "content": "{% extends \"base.html\" %}\n\n{% block title %}{% if is_instance %}Schema for all databases{% elif table_name %}Schema for {{ schemas[0].database }}.{{ table_name }}{% else %}Schema for {{ schemas[0].database }}{% endif %}{% endblock %}\n\n{% block body_class %}schema{% endblock %}\n\n{% block crumbs %}\n{% if is_instance %}\n{{ crumbs.nav(request=request) }}\n{% elif table_name %}\n{{ crumbs.nav(request=request, database=schemas[0].database, table=table_name) }}\n{% else %}\n{{ crumbs.nav(request=request, database=schemas[0].database) }}\n{% endif %}\n{% endblock %}\n\n{% block content %}\n
    \n    
    {% if is_instance %}Schema for all databases{% elif table_name %}Schema for {{ table_name }}{% else %}Schema for {{ schemas[0].database }}{% endif %}
    \n
    \n\n{% for item in schemas %}\n    {% if is_instance %}\n        
    {{ item.database }}
    \n    {% endif %}\n\n    {% if item.schema %}\n        
    {{ item.schema }}
    \n    {% else %}\n        
    No schema available for this database.
    \n    {% endif %}\n\n    {% if not loop.last %}\n        
    \n    {% endif %}\n{% endfor %}\n\n{% if not schemas %}\n    
    No databases with viewable schemas found.
    \n{% endif %}\n{% endblock %}\n"
  },
  {
    "path": "datasette/templates/show_json.html",
    "content": "{% extends \"base.html\" %}\n\n{% block title %}{{ filename }}{% endblock %}\n\n{% block body_class %}show-json{% endblock %}\n\n{% block content %}\n
    {{ filename }}
    \n\n
    {{ data_json }}
    \n\n{% endblock %}\n"
  },
  {
    "path": "datasette/templates/table.html",
    "content": "{% extends \"base.html\" %}\n\n{% block title %}{{ database }}: {{ table }}: {% if count or count == 0 %}{{ \"{:,}\".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %}{% if human_description_en %} {{ human_description_en }}{% endif %}{% endblock %}\n\n{% block extra_head %}\n{{- super() -}}\n\n\n\n\n\n{% endblock %}\n\n{% block body_class %}table db-{{ database|to_css_class }} table-{{ table|to_css_class }}{% endblock %}\n\n{% block crumbs %}\n{{ crumbs.nav(request=request, database=database, table=table) }}\n{% endblock %}\n\n{% block content %}\n
    \n    
    {{ metadata.get(\"title\") or table }}{% if is_view %} (view){% endif %}{% if private %} 🔒{% endif %}
    \n
    \n{% set action_links, action_title = actions(), \"View actions\" if is_view else \"Table actions\" %}\n{% include \"_action_menu.html\" %}\n\n{{ top_table() }}\n\n{% block description_source_license %}{% include \"_description_source_license.html\" %}{% endblock %}\n\n{% if metadata.get(\"columns\") %}\n
    \n    {% for column_name, column_description in metadata.columns.items() %}\n        
    {{ column_name }}
    {{ column_description }}
    \n    {% endfor %}\n
    \n{% endif %}\n\n{% if count or human_description_en %}\n    
    \n        {% if count == count_limit + 1 %}>{{ \"{:,}\".format(count_limit) }} rows\n        {% if allow_execute_sql and query.sql %} count all{% endif %}\n        {% elif count or count == 0 %}{{ \"{:,}\".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %}\n        {% if human_description_en %}{{ human_description_en }}{% endif %}\n    
    \n{% endif %}\n\n
    \n    {% if supports_search %}\n        
    \n    {% endif %}\n    {% for column, lookup, value in filters.selections() %}\n        
    \n            
    \n                \n            
    \n                \n            
    \n        
    \n    {% endfor %}\n    
    \n        
    \n            \n        
    \n            \n        
    \n    
    \n    
    \n        {% if is_sortable %}\n            
    \n                \n            
    \n            \n        {% endif %}\n        {% for key, value in form_hidden_args %}\n            \n        {% endfor %}\n        \n    
    \n
    \n\n{% if extra_wheres_for_ui %}\n
    \n    
    {{ extra_wheres_for_ui|length }} extra where clause{% if extra_wheres_for_ui|length != 1 %}s{% endif %}
    \n    
      \n    {% for extra_where in extra_wheres_for_ui %}\n        
    • {{ extra_where.text }} [remove]
      • \n    {% endfor %}\n    
    \n
    \n{% endif %}\n\n{% if query.sql and allow_execute_sql %}\n    
    ✎ View and edit SQL
    \n{% endif %}\n\n
    This data as {% for name, url in renderers.items() %}{{ name }}{{ \", \" if not loop.last }}{% endfor %}{% if display_rows %}, CSV (advanced){% endif %}
    \n\n{% if suggested_facets %}\n    {% include \"_suggested_facets.html\" %}\n{% endif %}\n\n{% if facets_timed_out %}\n    
    These facets timed out: {{ \", \".join(facets_timed_out) }}
    \n{% endif %}\n\n{% if facet_results %}\n    {% include \"_facet_results.html\" %}\n{% endif %}\n\n{% if all_columns %}\n\n\n{% if display_rows %}\n\n{% endif %}\n\n{% endif %}\n{% if set_column_type_ui %}\n\n{% endif %}\n\n{% include custom_table_templates %}\n\n{% if next_url %}\n     
    Next page
    \n{% endif %}\n\n{% if display_rows %}\n    
    \n        
    Advanced export
    \n        
    JSON shape:\n            default,\n            array,\n            newline-delimited{% if primary_keys %},\n                object\n            {% endif %}\n        
    \n        
    \n            
    \n                CSV options:\n                \n                {% if expandable_columns %}{% endif %}\n                {% if next_url and settings.allow_csv_stream %}{% endif %}\n                \n                {% for key, value in url_csv_hidden_args %}\n                    \n                {% endfor %}\n            
    \n        
    \n    
    \n{% endif %}\n\n{% if table_definition %}\n    
    {{ table_definition }}
    \n{% endif %}\n\n{% if view_definition %}\n    
    {{ view_definition }}
    \n{% endif %}\n\n{% if allow_execute_sql and query.sql %}\n\n{% endif %}\n\n{% endblock %}\n"
  },
  {
    "path": "datasette/tokens.py",
    "content": "\"\"\"\nToken handler system for Datasette.\n\nProvides a base class for token handlers and the default signed token handler.\nPlugins can implement register_token_handler to provide custom token backends\n(e.g. database-backed tokens that can be revoked and audited).\n\"\"\"\n\nfrom __future__ import annotations\n\nimport dataclasses\nimport time\nfrom typing import TYPE_CHECKING, Optional\n\nimport itsdangerous\n\nif TYPE_CHECKING:\n    from datasette.app import Datasette\n\n\n@dataclasses.dataclass\nclass TokenRestrictions:\n    \"\"\"\n    Restrictions to apply to a token, limiting which actions it can perform.\n\n    Use the builder methods to construct restrictions::\n\n        restrictions = (TokenRestrictions()\n            .allow_all(\"view-instance\")\n            .allow_database(\"mydb\", \"create-table\")\n            .allow_resource(\"mydb\", \"mytable\", \"insert-row\"))\n    \"\"\"\n\n    all: list[str] = dataclasses.field(default_factory=list)\n    database: dict[str, list[str]] = dataclasses.field(default_factory=dict)\n    resource: dict[str, dict[str, list[str]]] = dataclasses.field(default_factory=dict)\n\n    def allow_all(self, action: str) -> \"TokenRestrictions\":\n        \"\"\"Allow an action across all databases and resources.\"\"\"\n        self.all.append(action)\n        return self\n\n    def allow_database(self, database: str, action: str) -> \"TokenRestrictions\":\n        \"\"\"Allow an action on a specific database.\"\"\"\n        self.database.setdefault(database, []).append(action)\n        return self\n\n    def allow_resource(\n        self, database: str, resource: str, action: str\n    ) -> \"TokenRestrictions\":\n        \"\"\"Allow an action on a specific resource within a database.\"\"\"\n        self.resource.setdefault(database, {}).setdefault(resource, []).append(action)\n        return self\n\n\nclass TokenHandler:\n    \"\"\"\n    Base class for token handlers.\n\n    Subclass this and implement create_token() and verify_token() to provide\n    a custom token backend. Return an instance from the register_token_handler hook.\n    \"\"\"\n\n    name: str = \"\"\n\n    async def create_token(\n        self,\n        datasette: \"Datasette\",\n        actor_id: str,\n        *,\n        expires_after: Optional[int] = None,\n        restrictions: Optional[TokenRestrictions] = None,\n    ) -> str:\n        \"\"\"Create and return a token string for the given actor.\"\"\"\n        raise NotImplementedError\n\n    async def verify_token(self, datasette: \"Datasette\", token: str) -> Optional[dict]:\n        \"\"\"\n        Verify a token and return an actor dict, or None if this handler\n        does not recognize the token.\n        \"\"\"\n        raise NotImplementedError\n\n\nclass SignedTokenHandler(TokenHandler):\n    \"\"\"\n    Default token handler using itsdangerous signed tokens (dstok_ prefix).\n    \"\"\"\n\n    name = \"signed\"\n\n    async def create_token(\n        self,\n        datasette: \"Datasette\",\n        actor_id: str,\n        *,\n        expires_after: Optional[int] = None,\n        restrictions: Optional[TokenRestrictions] = None,\n    ) -> str:\n        if not datasette.setting(\"allow_signed_tokens\"):\n            raise ValueError(\n                \"Signed tokens are not enabled for this Datasette instance\"\n            )\n\n        token = {\"a\": actor_id, \"t\": int(time.time())}\n\n        def abbreviate_action(action):\n            action_obj = datasette.actions.get(action)\n            if not action_obj:\n                return action\n            return action_obj.abbr or action\n\n        if expires_after:\n            token[\"d\"] = expires_after\n        if restrictions and (\n            restrictions.all or restrictions.database or restrictions.resource\n        ):\n            token[\"_r\"] = {}\n            if restrictions.all:\n                token[\"_r\"][\"a\"] = [abbreviate_action(a) for a in restrictions.all]\n            if restrictions.database:\n                token[\"_r\"][\"d\"] = {}\n                for database, actions in restrictions.database.items():\n                    token[\"_r\"][\"d\"][database] = [abbreviate_action(a) for a in actions]\n            if restrictions.resource:\n                token[\"_r\"][\"r\"] = {}\n                for database, resources in restrictions.resource.items():\n                    for resource, actions in resources.items():\n                        token[\"_r\"][\"r\"].setdefault(database, {})[resource] = [\n                            abbreviate_action(a) for a in actions\n                        ]\n        return \"dstok_{}\".format(datasette.sign(token, namespace=\"token\"))\n\n    async def verify_token(self, datasette: \"Datasette\", token: str) -> Optional[dict]:\n        prefix = \"dstok_\"\n\n        if not datasette.setting(\"allow_signed_tokens\"):\n            return None\n\n        max_signed_tokens_ttl = datasette.setting(\"max_signed_tokens_ttl\")\n\n        if not token.startswith(prefix):\n            return None\n\n        raw = token[len(prefix) :]\n        try:\n            decoded = datasette.unsign(raw, namespace=\"token\")\n        except itsdangerous.BadSignature:\n            return None\n\n        if \"t\" not in decoded:\n            return None\n        created = decoded[\"t\"]\n        if not isinstance(created, int):\n            return None\n\n        duration = decoded.get(\"d\")\n        if duration is not None and not isinstance(duration, int):\n            return None\n\n        if (duration is None and max_signed_tokens_ttl) or (\n            duration is not None\n            and max_signed_tokens_ttl\n            and duration > max_signed_tokens_ttl\n        ):\n            duration = max_signed_tokens_ttl\n\n        if duration:\n            if time.time() - created > duration:\n                return None\n\n        actor = {\"id\": decoded[\"a\"], \"token\": \"dstok\"}\n\n        if \"_r\" in decoded:\n            actor[\"_r\"] = decoded[\"_r\"]\n\n        if duration:\n            actor[\"token_expires\"] = created + duration\n\n        return actor\n"
  },
  {
    "path": "datasette/tracer.py",
    "content": "import asyncio\nfrom contextlib import contextmanager\nfrom contextvars import ContextVar\nfrom markupsafe import escape\nimport time\nimport json\nimport traceback\n\ntracers = {}\n\nTRACE_RESERVED_KEYS = {\"type\", \"start\", \"end\", \"duration_ms\", \"traceback\"}\n\ntrace_task_id = ContextVar(\"trace_task_id\", default=None)\n\n\ndef get_task_id():\n    current = trace_task_id.get(None)\n    if current is not None:\n        return current\n    try:\n        loop = asyncio.get_event_loop()\n    except RuntimeError:\n        return None\n    return id(asyncio.current_task(loop=loop))\n\n\n@contextmanager\ndef trace_child_tasks():\n    token = trace_task_id.set(get_task_id())\n    yield\n    trace_task_id.reset(token)\n\n\n@contextmanager\ndef trace(trace_type, **kwargs):\n    assert not TRACE_RESERVED_KEYS.intersection(\n        kwargs.keys()\n    ), f\".trace() keyword parameters cannot include {TRACE_RESERVED_KEYS}\"\n    task_id = get_task_id()\n    if task_id is None:\n        yield kwargs\n        return\n    tracer = tracers.get(task_id)\n    if tracer is None:\n        yield kwargs\n        return\n    start = time.perf_counter()\n    captured_error = None\n    try:\n        yield kwargs\n    except Exception as ex:\n        captured_error = ex\n        raise\n    finally:\n        end = time.perf_counter()\n        trace_info = {\n            \"type\": trace_type,\n            \"start\": start,\n            \"end\": end,\n            \"duration_ms\": (end - start) * 1000,\n            \"traceback\": traceback.format_list(traceback.extract_stack(limit=6)[:-3]),\n            \"error\": str(captured_error) if captured_error else None,\n        }\n        trace_info.update(kwargs)\n        tracer.append(trace_info)\n\n\n@contextmanager\ndef capture_traces(tracer):\n    # tracer is a list\n    task_id = get_task_id()\n    if task_id is None:\n        yield\n        return\n    tracers[task_id] = tracer\n    yield\n    del tracers[task_id]\n\n\nclass AsgiTracer:\n    # If the body is larger than this we don't attempt to append the trace\n    max_body_bytes = 1024 * 256  # 256 KB\n\n    def __init__(self, app):\n        self.app = app\n\n    async def __call__(self, scope, receive, send):\n        if b\"_trace=1\" not in scope.get(\"query_string\", b\"\").split(b\"&\"):\n            await self.app(scope, receive, send)\n            return\n        trace_start = time.perf_counter()\n        traces = []\n\n        accumulated_body = b\"\"\n        size_limit_exceeded = False\n        response_headers = []\n\n        async def wrapped_send(message):\n            nonlocal accumulated_body, size_limit_exceeded, response_headers\n\n            if message[\"type\"] == \"http.response.start\":\n                response_headers = message[\"headers\"]\n                await send(message)\n                return\n\n            if message[\"type\"] != \"http.response.body\" or size_limit_exceeded:\n                await send(message)\n                return\n\n            # Accumulate body until the end or until size is exceeded\n            accumulated_body += message[\"body\"]\n            if len(accumulated_body) > self.max_body_bytes:\n                # Send what we have accumulated so far\n                await send(\n                    {\n                        \"type\": \"http.response.body\",\n                        \"body\": accumulated_body,\n                        \"more_body\": bool(message.get(\"more_body\")),\n                    }\n                )\n                size_limit_exceeded = True\n                return\n\n            if not message.get(\"more_body\"):\n                # We have all the body - modify it and send the result\n                # TODO: What to do about Content-Type or other cases?\n                trace_info = {\n                    \"request_duration_ms\": 1000 * (time.perf_counter() - trace_start),\n                    \"sum_trace_duration_ms\": sum(t[\"duration_ms\"] for t in traces),\n                    \"num_traces\": len(traces),\n                    \"traces\": traces,\n                }\n                try:\n                    content_type = [\n                        v.decode(\"utf8\")\n                        for k, v in response_headers\n                        if k.lower() == b\"content-type\"\n                    ][0]\n                except IndexError:\n                    content_type = \"\"\n                if \"text/html\" in content_type and b\"\" in accumulated_body:\n                    extra = escape(json.dumps(trace_info, indent=2))\n                    extra_html = f\"
    {extra}
    \".encode(\"utf8\")\n                    accumulated_body = accumulated_body.replace(b\"\", extra_html)\n                elif \"json\" in content_type and accumulated_body.startswith(b\"{\"):\n                    data = json.loads(accumulated_body.decode(\"utf8\"))\n                    if \"_trace\" not in data:\n                        data[\"_trace\"] = trace_info\n                    accumulated_body = json.dumps(data).encode(\"utf8\")\n                await send({\"type\": \"http.response.body\", \"body\": accumulated_body})\n\n        with capture_traces(traces):\n            await self.app(scope, receive, wrapped_send)\n"
  },
  {
    "path": "datasette/url_builder.py",
    "content": "from .utils import tilde_encode, path_with_format, PrefixedUrlString\nimport urllib\n\n\nclass Urls:\n    def __init__(self, ds):\n        self.ds = ds\n\n    def path(self, path, format=None):\n        if not isinstance(path, PrefixedUrlString):\n            if path.startswith(\"/\"):\n                path = path[1:]\n            path = self.ds.setting(\"base_url\") + path\n        if format is not None:\n            path = path_with_format(path=path, format=format)\n        return PrefixedUrlString(path)\n\n    def instance(self, format=None):\n        return self.path(\"\", format=format)\n\n    def static(self, path):\n        return self.path(f\"-/static/{path}\")\n\n    def static_plugins(self, plugin, path):\n        return self.path(f\"-/static-plugins/{plugin}/{path}\")\n\n    def logout(self):\n        return self.path(\"-/logout\")\n\n    def database(self, database, format=None):\n        db = self.ds.get_database(database)\n        return self.path(tilde_encode(db.route), format=format)\n\n    def database_query(self, database, sql, format=None):\n        path = f\"{self.database(database)}/-/query?\" + urllib.parse.urlencode(\n            {\"sql\": sql}\n        )\n        return self.path(path, format=format)\n\n    def table(self, database, table, format=None):\n        path = f\"{self.database(database)}/{tilde_encode(table)}\"\n        if format is not None:\n            path = path_with_format(path=path, format=format)\n        return PrefixedUrlString(path)\n\n    def query(self, database, query, format=None):\n        path = f\"{self.database(database)}/{tilde_encode(query)}\"\n        if format is not None:\n            path = path_with_format(path=path, format=format)\n        return PrefixedUrlString(path)\n\n    def row(self, database, table, row_path, format=None):\n        path = f\"{self.table(database, table)}/{row_path}\"\n        if format is not None:\n            path = path_with_format(path=path, format=format)\n        return PrefixedUrlString(path)\n\n    def row_blob(self, database, table, row_path, column):\n        return self.table(database, table) + \"/{}.blob?_blob_column={}\".format(\n            row_path, urllib.parse.quote_plus(column)\n        )\n"
  },
  {
    "path": "datasette/utils/__init__.py",
    "content": "import asyncio\nfrom contextlib import contextmanager\nimport aiofiles\nimport click\nfrom collections import OrderedDict, namedtuple, Counter\nimport copy\nimport dataclasses\nimport base64\nimport hashlib\nimport inspect\nimport json\nimport markupsafe\nimport mergedeep\nimport os\nimport re\nimport shlex\nimport tempfile\nimport typing\nimport time\nimport types\nimport secrets\nimport shutil\nfrom typing import Iterable, List, Tuple\nimport urllib\nimport yaml\nfrom .shutil_backport import copytree\nfrom .sqlite import sqlite3, supports_table_xinfo\n\nif typing.TYPE_CHECKING:\n    from datasette.database import Database\n    from datasette.permissions import Resource\n\n\n@dataclasses.dataclass\nclass PaginatedResources:\n    \"\"\"Paginated results from allowed_resources query.\"\"\"\n\n    resources: List[\"Resource\"]\n    next: str | None  # Keyset token for next page (None if no more results)\n    _datasette: typing.Any = dataclasses.field(default=None, repr=False)\n    _action: str = dataclasses.field(default=None, repr=False)\n    _actor: typing.Any = dataclasses.field(default=None, repr=False)\n    _parent: str | None = dataclasses.field(default=None, repr=False)\n    _include_is_private: bool = dataclasses.field(default=False, repr=False)\n    _include_reasons: bool = dataclasses.field(default=False, repr=False)\n    _limit: int = dataclasses.field(default=100, repr=False)\n\n    async def all(self):\n        \"\"\"\n        Async generator that yields all resources across all pages.\n\n        Automatically handles pagination under the hood. This is useful when you need\n        to iterate through all results without manually managing pagination tokens.\n\n        Yields:\n            Resource objects one at a time\n\n        Example:\n            page = await datasette.allowed_resources(\"view-table\", actor)\n            async for table in page.all():\n                print(f\"{table.parent}/{table.child}\")\n        \"\"\"\n        # Yield all resources from current page\n        for resource in self.resources:\n            yield resource\n\n        # Continue fetching subsequent pages if there are more\n        next_token = self.next\n        while next_token:\n            page = await self._datasette.allowed_resources(\n                self._action,\n                self._actor,\n                parent=self._parent,\n                include_is_private=self._include_is_private,\n                include_reasons=self._include_reasons,\n                limit=self._limit,\n                next=next_token,\n            )\n            for resource in page.resources:\n                yield resource\n            next_token = page.next\n\n\n# From https://www.sqlite.org/lang_keywords.html\nreserved_words = set(\n    (\n        \"abort action add after all alter analyze and as asc attach autoincrement \"\n        \"before begin between by cascade case cast check collate column commit \"\n        \"conflict constraint create cross current_date current_time \"\n        \"current_timestamp database default deferrable deferred delete desc detach \"\n        \"distinct drop each else end escape except exclusive exists explain fail \"\n        \"for foreign from full glob group having if ignore immediate in index \"\n        \"indexed initially inner insert instead intersect into is isnull join key \"\n        \"left like limit match natural no not notnull null of offset on or order \"\n        \"outer plan pragma primary query raise recursive references regexp reindex \"\n        \"release rename replace restrict right rollback row savepoint select set \"\n        \"table temp temporary then to transaction trigger union unique update using \"\n        \"vacuum values view virtual when where with without\"\n    ).split()\n)\n\nAPT_GET_DOCKERFILE_EXTRAS = r\"\"\"\nRUN apt-get update && \\\n    apt-get install -y {} && \\\n    rm -rf /var/lib/apt/lists/*\n\"\"\"\n\n# Can replace with sqlite-utils when I add that dependency\nSPATIALITE_PATHS = (\n    \"/usr/lib/x86_64-linux-gnu/mod_spatialite.so\",\n    \"/usr/local/lib/mod_spatialite.dylib\",\n    \"/usr/local/lib/mod_spatialite.so\",\n    \"/opt/homebrew/lib/mod_spatialite.dylib\",\n)\n# Used to display /-/versions.json SpatiaLite information\nSPATIALITE_FUNCTIONS = (\n    \"spatialite_version\",\n    \"spatialite_target_cpu\",\n    \"check_strict_sql_quoting\",\n    \"freexl_version\",\n    \"proj_version\",\n    \"geos_version\",\n    \"rttopo_version\",\n    \"libxml2_version\",\n    \"HasIconv\",\n    \"HasMathSQL\",\n    \"HasGeoCallbacks\",\n    \"HasProj\",\n    \"HasProj6\",\n    \"HasGeos\",\n    \"HasGeosAdvanced\",\n    \"HasGeosTrunk\",\n    \"HasGeosReentrant\",\n    \"HasGeosOnlyReentrant\",\n    \"HasMiniZip\",\n    \"HasRtTopo\",\n    \"HasLibXML2\",\n    \"HasEpsg\",\n    \"HasFreeXL\",\n    \"HasGeoPackage\",\n    \"HasGCP\",\n    \"HasTopology\",\n    \"HasKNN\",\n    \"HasRouting\",\n)\n# Length of hash subset used in hashed URLs:\nHASH_LENGTH = 7\n\n\n# Can replace this with Column from sqlite_utils when I add that dependency\nColumn = namedtuple(\n    \"Column\", (\"cid\", \"name\", \"type\", \"notnull\", \"default_value\", \"is_pk\", \"hidden\")\n)\n\nfunctions_marked_as_documented = []\n\n\ndef documented(fn):\n    functions_marked_as_documented.append(fn)\n    return fn\n\n\n@documented\nasync def await_me_maybe(value: typing.Any) -> typing.Any:\n    \"If value is callable, call it. If awaitable, await it. Otherwise return it.\"\n    if callable(value):\n        value = value()\n    if asyncio.iscoroutine(value):\n        value = await value\n    return value\n\n\ndef urlsafe_components(token):\n    \"\"\"Splits token on commas and tilde-decodes each component\"\"\"\n    return [tilde_decode(b) for b in token.split(\",\")]\n\n\ndef path_from_row_pks(row, pks, use_rowid, quote=True):\n    \"\"\"Generate an optionally tilde-encoded unique identifier\n    for a row from its primary keys.\"\"\"\n    if use_rowid:\n        bits = [row[\"rowid\"]]\n    else:\n        bits = [\n            row[pk][\"value\"] if isinstance(row[pk], dict) else row[pk] for pk in pks\n        ]\n    if quote:\n        bits = [tilde_encode(str(bit)) for bit in bits]\n    else:\n        bits = [str(bit) for bit in bits]\n\n    return \",\".join(bits)\n\n\ndef compound_keys_after_sql(pks, start_index=0):\n    # Implementation of keyset pagination\n    # See https://github.com/simonw/datasette/issues/190\n    # For pk1/pk2/pk3 returns:\n    #\n    # ([pk1] > :p0)\n    #   or\n    # ([pk1] = :p0 and [pk2] > :p1)\n    #   or\n    # ([pk1] = :p0 and [pk2] = :p1 and [pk3] > :p2)\n    or_clauses = []\n    pks_left = pks[:]\n    while pks_left:\n        and_clauses = []\n        last = pks_left[-1]\n        rest = pks_left[:-1]\n        and_clauses = [\n            f\"{escape_sqlite(pk)} = :p{i + start_index}\" for i, pk in enumerate(rest)\n        ]\n        and_clauses.append(f\"{escape_sqlite(last)} > :p{len(rest) + start_index}\")\n        or_clauses.append(f\"({' and '.join(and_clauses)})\")\n        pks_left.pop()\n    or_clauses.reverse()\n    return \"({})\".format(\"\\n  or\\n\".join(or_clauses))\n\n\nclass CustomJSONEncoder(json.JSONEncoder):\n    def default(self, obj):\n        if isinstance(obj, sqlite3.Row):\n            return tuple(obj)\n        if isinstance(obj, sqlite3.Cursor):\n            return list(obj)\n        if isinstance(obj, bytes):\n            # Does it encode to utf8?\n            try:\n                return obj.decode(\"utf8\")\n            except UnicodeDecodeError:\n                return {\n                    \"$base64\": True,\n                    \"encoded\": base64.b64encode(obj).decode(\"latin1\"),\n                }\n        return json.JSONEncoder.default(self, obj)\n\n\n@contextmanager\ndef sqlite_timelimit(conn, ms):\n    deadline = time.perf_counter() + (ms / 1000)\n    # n is the number of SQLite virtual machine instructions that will be\n    # executed between each check. It takes about 0.08ms to execute 1000.\n    # https://github.com/simonw/datasette/issues/1679\n    n = 1000\n    if ms <= 20:\n        # This mainly happens while executing our test suite\n        n = 1\n\n    def handler():\n        if time.perf_counter() >= deadline:\n            # Returning 1 terminates the query with an error\n            return 1\n\n    conn.set_progress_handler(handler, n)\n    try:\n        yield\n    finally:\n        conn.set_progress_handler(None, n)\n\n\nclass InvalidSql(Exception):\n    pass\n\n\n# Allow SQL to start with a /* */ or -- comment\ncomment_re = (\n    # Start of string, then any amount of whitespace\n    r\"^\\s*(\"\n    +\n    # Comment that starts with -- and ends at a newline\n    r\"(?:\\-\\-.*?\\n\\s*)\"\n    +\n    # Comment that starts with /* and ends with */ - but does not have */ in it\n    r\"|(?:\\/\\*((?!\\*\\/)[\\s\\S])*\\*\\/)\"\n    +\n    # Whitespace\n    r\"\\s*)*\\s*\"\n)\n\nallowed_sql_res = [\n    re.compile(comment_re + r\"select\\b\"),\n    re.compile(comment_re + r\"explain\\s+select\\b\"),\n    re.compile(comment_re + r\"explain\\s+query\\s+plan\\s+select\\b\"),\n    re.compile(comment_re + r\"with\\b\"),\n    re.compile(comment_re + r\"explain\\s+with\\b\"),\n    re.compile(comment_re + r\"explain\\s+query\\s+plan\\s+with\\b\"),\n]\n\nallowed_pragmas = (\n    \"database_list\",\n    \"foreign_key_list\",\n    \"function_list\",\n    \"index_info\",\n    \"index_list\",\n    \"index_xinfo\",\n    \"page_count\",\n    \"max_page_count\",\n    \"page_size\",\n    \"schema_version\",\n    \"table_info\",\n    \"table_xinfo\",\n    \"table_list\",\n)\ndisallawed_sql_res = [\n    (\n        re.compile(f\"pragma(?!_({'|'.join(allowed_pragmas)}))\"),\n        \"Statement contained a disallowed PRAGMA. Allowed pragma functions are {}\".format(\n            \", \".join(\"pragma_{}()\".format(pragma) for pragma in allowed_pragmas)\n        ),\n    )\n]\n\n\ndef validate_sql_select(sql):\n    sql = \"\\n\".join(\n        line for line in sql.split(\"\\n\") if not line.strip().startswith(\"--\")\n    )\n    sql = sql.strip().lower()\n    if not any(r.match(sql) for r in allowed_sql_res):\n        raise InvalidSql(\"Statement must be a SELECT\")\n    for r, msg in disallawed_sql_res:\n        if r.search(sql):\n            raise InvalidSql(msg)\n\n\ndef append_querystring(url, querystring):\n    op = \"&\" if (\"?\" in url) else \"?\"\n    return f\"{url}{op}{querystring}\"\n\n\ndef path_with_added_args(request, args, path=None):\n    path = path or request.path\n    if isinstance(args, dict):\n        args = args.items()\n    args_to_remove = {k for k, v in args if v is None}\n    current = []\n    for key, value in urllib.parse.parse_qsl(request.query_string):\n        if key not in args_to_remove:\n            current.append((key, value))\n    current.extend([(key, value) for key, value in args if value is not None])\n    query_string = urllib.parse.urlencode(current)\n    if query_string:\n        query_string = f\"?{query_string}\"\n    return path + query_string\n\n\ndef path_with_removed_args(request, args, path=None):\n    query_string = request.query_string\n    if path is None:\n        path = request.path\n    else:\n        if \"?\" in path:\n            bits = path.split(\"?\", 1)\n            path, query_string = bits\n    # args can be a dict or a set\n    current = []\n    if isinstance(args, set):\n\n        def should_remove(key, value):\n            return key in args\n\n    elif isinstance(args, dict):\n        # Must match key AND value\n        def should_remove(key, value):\n            return args.get(key) == value\n\n    for key, value in urllib.parse.parse_qsl(query_string):\n        if not should_remove(key, value):\n            current.append((key, value))\n    query_string = urllib.parse.urlencode(current)\n    if query_string:\n        query_string = f\"?{query_string}\"\n    return path + query_string\n\n\ndef path_with_replaced_args(request, args, path=None):\n    path = path or request.path\n    if isinstance(args, dict):\n        args = args.items()\n    keys_to_replace = {p[0] for p in args}\n    current = []\n    for key, value in urllib.parse.parse_qsl(request.query_string):\n        if key not in keys_to_replace:\n            current.append((key, value))\n    current.extend([p for p in args if p[1] is not None])\n    query_string = urllib.parse.urlencode(current)\n    if query_string:\n        query_string = f\"?{query_string}\"\n    return path + query_string\n\n\n_css_re = re.compile(r\"\"\"['\"\\n\\\\]\"\"\")\n_boring_keyword_re = re.compile(r\"^[a-zA-Z_][a-zA-Z0-9_]*$\")\n\n\ndef escape_css_string(s):\n    return _css_re.sub(\n        lambda m: \"\\\\\" + (f\"{ord(m.group()):X}\".zfill(6)),\n        s.replace(\"\\r\\n\", \"\\n\"),\n    )\n\n\ndef escape_sqlite(s):\n    if _boring_keyword_re.match(s) and (s.lower() not in reserved_words):\n        return s\n    else:\n        return f\"[{s}]\"\n\n\ndef make_dockerfile(\n    files,\n    metadata_file,\n    extra_options,\n    branch,\n    template_dir,\n    plugins_dir,\n    static,\n    install,\n    spatialite,\n    version_note,\n    secret,\n    environment_variables=None,\n    port=8001,\n    apt_get_extras=None,\n):\n    cmd = [\"datasette\", \"serve\", \"--host\", \"0.0.0.0\"]\n    environment_variables = environment_variables or {}\n    environment_variables[\"DATASETTE_SECRET\"] = secret\n    apt_get_extras = apt_get_extras or []\n    for filename in files:\n        cmd.extend([\"-i\", filename])\n    cmd.extend([\"--cors\", \"--inspect-file\", \"inspect-data.json\"])\n    if metadata_file:\n        cmd.extend([\"--metadata\", f\"{metadata_file}\"])\n    if template_dir:\n        cmd.extend([\"--template-dir\", \"templates/\"])\n    if plugins_dir:\n        cmd.extend([\"--plugins-dir\", \"plugins/\"])\n    if version_note:\n        cmd.extend([\"--version-note\", f\"{version_note}\"])\n    if static:\n        for mount_point, _ in static:\n            cmd.extend([\"--static\", f\"{mount_point}:{mount_point}\"])\n    if extra_options:\n        for opt in extra_options.split():\n            cmd.append(f\"{opt}\")\n    cmd = [shlex.quote(part) for part in cmd]\n    # port attribute is a (fixed) env variable and should not be quoted\n    cmd.extend([\"--port\", \"$PORT\"])\n    cmd = \" \".join(cmd)\n    if branch:\n        install = [f\"https://github.com/simonw/datasette/archive/{branch}.zip\"] + list(\n            install\n        )\n    else:\n        install = [\"datasette\"] + list(install)\n\n    apt_get_extras_ = []\n    apt_get_extras_.extend(apt_get_extras)\n    apt_get_extras = apt_get_extras_\n    if spatialite:\n        apt_get_extras.extend([\"python3-dev\", \"gcc\", \"libsqlite3-mod-spatialite\"])\n        environment_variables[\"SQLITE_EXTENSIONS\"] = (\n            \"/usr/lib/x86_64-linux-gnu/mod_spatialite.so\"\n        )\n    return \"\"\"\nFROM python:3.11.0-slim-bullseye\nCOPY . /app\nWORKDIR /app\n{apt_get_extras}\n{environment_variables}\nRUN pip install -U {install_from}\nRUN datasette inspect {files} --inspect-file inspect-data.json\nENV PORT {port}\nEXPOSE {port}\nCMD {cmd}\"\"\".format(\n        apt_get_extras=(\n            APT_GET_DOCKERFILE_EXTRAS.format(\" \".join(apt_get_extras))\n            if apt_get_extras\n            else \"\"\n        ),\n        environment_variables=\"\\n\".join(\n            [\n                \"ENV {} '{}'\".format(key, value)\n                for key, value in environment_variables.items()\n            ]\n        ),\n        install_from=\" \".join(install),\n        files=\" \".join(files),\n        port=port,\n        cmd=cmd,\n    ).strip()\n\n\n@contextmanager\ndef temporary_docker_directory(\n    files,\n    name,\n    metadata,\n    extra_options,\n    branch,\n    template_dir,\n    plugins_dir,\n    static,\n    install,\n    spatialite,\n    version_note,\n    secret,\n    extra_metadata=None,\n    environment_variables=None,\n    port=8001,\n    apt_get_extras=None,\n):\n    extra_metadata = extra_metadata or {}\n    tmp = tempfile.TemporaryDirectory()\n    # We create a datasette folder in there to get a nicer now deploy name\n    datasette_dir = os.path.join(tmp.name, name)\n    os.mkdir(datasette_dir)\n    saved_cwd = os.getcwd()\n    file_paths = [os.path.join(saved_cwd, file_path) for file_path in files]\n    file_names = [os.path.split(f)[-1] for f in files]\n    if metadata:\n        metadata_content = parse_metadata(metadata.read())\n    else:\n        metadata_content = {}\n    # Merge in the non-null values in extra_metadata\n    mergedeep.merge(\n        metadata_content,\n        {key: value for key, value in extra_metadata.items() if value is not None},\n    )\n    try:\n        dockerfile = make_dockerfile(\n            file_names,\n            metadata_content and \"metadata.json\",\n            extra_options,\n            branch,\n            template_dir,\n            plugins_dir,\n            static,\n            install,\n            spatialite,\n            version_note,\n            secret,\n            environment_variables,\n            port=port,\n            apt_get_extras=apt_get_extras,\n        )\n        os.chdir(datasette_dir)\n        if metadata_content:\n            with open(\"metadata.json\", \"w\") as fp:\n                fp.write(json.dumps(metadata_content, indent=2))\n        with open(\"Dockerfile\", \"w\") as fp:\n            fp.write(dockerfile)\n        for path, filename in zip(file_paths, file_names):\n            link_or_copy(path, os.path.join(datasette_dir, filename))\n        if template_dir:\n            link_or_copy_directory(\n                os.path.join(saved_cwd, template_dir),\n                os.path.join(datasette_dir, \"templates\"),\n            )\n        if plugins_dir:\n            link_or_copy_directory(\n                os.path.join(saved_cwd, plugins_dir),\n                os.path.join(datasette_dir, \"plugins\"),\n            )\n        for mount_point, path in static:\n            link_or_copy_directory(\n                os.path.join(saved_cwd, path), os.path.join(datasette_dir, mount_point)\n            )\n        yield datasette_dir\n    finally:\n        tmp.cleanup()\n        os.chdir(saved_cwd)\n\n\ndef detect_primary_keys(conn, table):\n    \"\"\"Figure out primary keys for a table.\"\"\"\n    columns = table_column_details(conn, table)\n    pks = [column for column in columns if column.is_pk]\n    pks.sort(key=lambda column: column.is_pk)\n    return [column.name for column in pks]\n\n\ndef get_outbound_foreign_keys(conn, table):\n    infos = conn.execute(f\"PRAGMA foreign_key_list([{table}])\").fetchall()\n    fks = []\n    for info in infos:\n        if info is not None:\n            id, seq, table_name, from_, to_, on_update, on_delete, match = info\n            fks.append(\n                {\n                    \"column\": from_,\n                    \"other_table\": table_name,\n                    \"other_column\": to_,\n                    \"id\": id,\n                    \"seq\": seq,\n                }\n            )\n    # Filter out compound foreign keys by removing any where \"id\" is not unique\n    id_counts = Counter(fk[\"id\"] for fk in fks)\n    return [\n        {\n            \"column\": fk[\"column\"],\n            \"other_table\": fk[\"other_table\"],\n            \"other_column\": fk[\"other_column\"],\n        }\n        for fk in fks\n        if id_counts[fk[\"id\"]] == 1\n    ]\n\n\ndef get_all_foreign_keys(conn):\n    tables = [\n        r[0]\n        for r in conn.execute(\n            'select name from sqlite_master where type=\"table\" order by name'\n        )\n    ]\n    table_to_foreign_keys = {}\n    for table in tables:\n        table_to_foreign_keys[table] = {\"incoming\": [], \"outgoing\": []}\n    for table in tables:\n        fks = get_outbound_foreign_keys(conn, table)\n        for fk in fks:\n            table_name = fk[\"other_table\"]\n            from_ = fk[\"column\"]\n            to_ = fk[\"other_column\"]\n            if table_name not in table_to_foreign_keys:\n                # Weird edge case where something refers to a table that does\n                # not actually exist\n                continue\n            table_to_foreign_keys[table_name][\"incoming\"].append(\n                {\"other_table\": table, \"column\": to_, \"other_column\": from_}\n            )\n            table_to_foreign_keys[table][\"outgoing\"].append(\n                {\"other_table\": table_name, \"column\": from_, \"other_column\": to_}\n            )\n\n    # Sort foreign keys for deterministic ordering\n    for table in table_to_foreign_keys:\n        table_to_foreign_keys[table][\"incoming\"].sort(\n            key=lambda fk: (fk[\"other_table\"], fk[\"column\"], fk[\"other_column\"])\n        )\n        table_to_foreign_keys[table][\"outgoing\"].sort(\n            key=lambda fk: (fk[\"other_table\"], fk[\"column\"], fk[\"other_column\"])\n        )\n\n    return table_to_foreign_keys\n\n\ndef detect_spatialite(conn):\n    rows = conn.execute(\n        'select 1 from sqlite_master where tbl_name = \"geometry_columns\"'\n    ).fetchall()\n    return len(rows) > 0\n\n\ndef detect_fts(conn, table):\n    \"\"\"Detect if table has a corresponding FTS virtual table and return it\"\"\"\n    rows = conn.execute(detect_fts_sql(table)).fetchall()\n    if len(rows) == 0:\n        return None\n    else:\n        return rows[0][0]\n\n\ndef detect_fts_sql(table):\n    return r\"\"\"\n        select name from sqlite_master\n            where rootpage = 0\n            and (\n                sql like '%VIRTUAL TABLE%USING FTS%content=\"{table}\"%'\n                or sql like '%VIRTUAL TABLE%USING FTS%content=[{table}]%'\n                or (\n                    tbl_name = \"{table}\"\n                    and sql like '%VIRTUAL TABLE%USING FTS%'\n                )\n            )\n    \"\"\".format(table=table.replace(\"'\", \"''\"))\n\n\ndef detect_json1(conn=None):\n    if conn is None:\n        conn = sqlite3.connect(\":memory:\")\n    try:\n        conn.execute(\"SELECT json('{}')\")\n        return True\n    except Exception:\n        return False\n\n\ndef table_columns(conn, table):\n    return [column.name for column in table_column_details(conn, table)]\n\n\ndef table_column_details(conn, table):\n    if supports_table_xinfo():\n        # table_xinfo was added in 3.26.0\n        return [\n            Column(*r)\n            for r in conn.execute(\n                f\"PRAGMA table_xinfo({escape_sqlite(table)});\"\n            ).fetchall()\n        ]\n    else:\n        # First trigger a query against sqlite_master to fix an intermittent\n        # test failure, see https://github.com/simonw/datasette/issues/2632\n        conn.execute(\"select 1 from sqlite_master limit 1\").fetchall()\n        return [\n            # Treat hidden as 0 for all columns.\n            Column(*(list(r) + [0]))\n            for r in conn.execute(\n                f\"PRAGMA table_info({escape_sqlite(table)});\"\n            ).fetchall()\n        ]\n\n\nfilter_column_re = re.compile(r\"^_filter_column_\\d+$\")\n\n\ndef filters_should_redirect(special_args):\n    redirect_params = []\n    # Handle _filter_column=foo&_filter_op=exact&_filter_value=...\n    filter_column = special_args.get(\"_filter_column\")\n    filter_op = special_args.get(\"_filter_op\") or \"\"\n    filter_value = special_args.get(\"_filter_value\") or \"\"\n    if \"__\" in filter_op:\n        filter_op, filter_value = filter_op.split(\"__\", 1)\n    if filter_column:\n        redirect_params.append((f\"{filter_column}__{filter_op}\", filter_value))\n    for key in (\"_filter_column\", \"_filter_op\", \"_filter_value\"):\n        if key in special_args:\n            redirect_params.append((key, None))\n    # Now handle _filter_column_1=name&_filter_op_1=contains&_filter_value_1=hello\n    column_keys = [k for k in special_args if filter_column_re.match(k)]\n    for column_key in column_keys:\n        number = column_key.split(\"_\")[-1]\n        column = special_args[column_key]\n        op = special_args.get(f\"_filter_op_{number}\") or \"exact\"\n        value = special_args.get(f\"_filter_value_{number}\") or \"\"\n        if \"__\" in op:\n            op, value = op.split(\"__\", 1)\n        if column:\n            redirect_params.append((f\"{column}__{op}\", value))\n        redirect_params.extend(\n            [\n                (f\"_filter_column_{number}\", None),\n                (f\"_filter_op_{number}\", None),\n                (f\"_filter_value_{number}\", None),\n            ]\n        )\n    return redirect_params\n\n\nwhitespace_re = re.compile(r\"\\s\")\n\n\ndef is_url(value):\n    \"\"\"Must start with http:// or https:// and contain JUST a URL\"\"\"\n    if not isinstance(value, str):\n        return False\n    if not value.startswith(\"http://\") and not value.startswith(\"https://\"):\n        return False\n    # Any whitespace at all is invalid\n    if whitespace_re.search(value):\n        return False\n    return True\n\n\ncss_class_re = re.compile(r\"^[a-zA-Z]+[_a-zA-Z0-9-]*$\")\ncss_invalid_chars_re = re.compile(r\"[^a-zA-Z0-9_\\-]\")\n\n\ndef to_css_class(s):\n    \"\"\"\n    Given a string (e.g. a table name) returns a valid unique CSS class.\n    For simple cases, just returns the string again. If the string is not a\n    valid CSS class (we disallow - and _ prefixes even though they are valid\n    as they may be confused with browser prefixes) we strip invalid characters\n    and add a 6 char md5 sum suffix, to make sure two tables with identical\n    names after stripping characters don't end up with the same CSS class.\n    \"\"\"\n    if css_class_re.match(s):\n        return s\n    md5_suffix = md5_not_usedforsecurity(s)[:6]\n    # Strip leading _, -\n    s = s.lstrip(\"_\").lstrip(\"-\")\n    # Replace any whitespace with hyphens\n    s = \"-\".join(s.split())\n    # Remove any remaining invalid characters\n    s = css_invalid_chars_re.sub(\"\", s)\n    # Attach the md5 suffix\n    bits = [b for b in (s, md5_suffix) if b]\n    return \"-\".join(bits)\n\n\ndef link_or_copy(src, dst):\n    # Intended for use in populating a temp directory. We link if possible,\n    # but fall back to copying if the temp directory is on a different device\n    # https://github.com/simonw/datasette/issues/141\n    try:\n        os.link(src, dst)\n    except OSError:\n        shutil.copyfile(src, dst)\n\n\ndef link_or_copy_directory(src, dst):\n    try:\n        copytree(src, dst, copy_function=os.link, dirs_exist_ok=True)\n    except OSError:\n        copytree(src, dst, dirs_exist_ok=True)\n\n\ndef module_from_path(path, name):\n    # Adapted from http://sayspy.blogspot.com/2011/07/how-to-import-module-from-just-file.html\n    mod = types.ModuleType(name)\n    mod.__file__ = path\n    with open(path, \"r\") as file:\n        code = compile(file.read(), path, \"exec\", dont_inherit=True)\n    exec(code, mod.__dict__)\n    return mod\n\n\ndef path_with_format(\n    *, request=None, path=None, format=None, extra_qs=None, replace_format=None\n):\n    qs = extra_qs or {}\n    path = request.path if request else path\n    if replace_format and path.endswith(f\".{replace_format}\"):\n        path = path[: -(1 + len(replace_format))]\n    if \".\" in path:\n        qs[\"_format\"] = format\n    else:\n        path = f\"{path}.{format}\"\n    if qs:\n        extra = urllib.parse.urlencode(sorted(qs.items()))\n        if request and request.query_string:\n            path = f\"{path}?{request.query_string}&{extra}\"\n        else:\n            path = f\"{path}?{extra}\"\n    elif request and request.query_string:\n        path = f\"{path}?{request.query_string}\"\n    return path\n\n\nclass CustomRow(OrderedDict):\n    # Loose imitation of sqlite3.Row which offers\n    # both index-based AND key-based lookups\n    def __init__(self, columns, values=None):\n        self.columns = columns\n        if values:\n            self.update(values)\n\n    def __getitem__(self, key):\n        if isinstance(key, int):\n            return super().__getitem__(self.columns[key])\n        else:\n            return super().__getitem__(key)\n\n    def __iter__(self):\n        for column in self.columns:\n            yield self[column]\n\n\ndef value_as_boolean(value):\n    if value.lower() not in (\"on\", \"off\", \"true\", \"false\", \"1\", \"0\"):\n        raise ValueAsBooleanError\n    return value.lower() in (\"on\", \"true\", \"1\")\n\n\nclass ValueAsBooleanError(ValueError):\n    pass\n\n\nclass WriteLimitExceeded(Exception):\n    pass\n\n\nclass LimitedWriter:\n    def __init__(self, writer, limit_mb):\n        self.writer = writer\n        self.limit_bytes = limit_mb * 1024 * 1024\n        self.bytes_count = 0\n\n    async def write(self, bytes):\n        self.bytes_count += len(bytes)\n        if self.limit_bytes and (self.bytes_count > self.limit_bytes):\n            raise WriteLimitExceeded(f\"CSV contains more than {self.limit_bytes} bytes\")\n        await self.writer.write(bytes)\n\n\nclass EscapeHtmlWriter:\n    def __init__(self, writer):\n        self.writer = writer\n\n    async def write(self, content):\n        await self.writer.write(markupsafe.escape(content))\n\n\n_infinities = {float(\"inf\"), float(\"-inf\")}\n\n\ndef remove_infinites(row):\n    \"\"\"\n    Replace float('inf') and float('-inf') with None in a row.\n\n    Returns the original row object unchanged if no infinities are found.\n    \"\"\"\n    if isinstance(row, dict):\n        for v in row.values():\n            if isinstance(v, float) and v in _infinities:\n                return {\n                    k: (None if isinstance(v2, float) and v2 in _infinities else v2)\n                    for k, v2 in row.items()\n                }\n    else:\n        for v in row:\n            if isinstance(v, float) and v in _infinities:\n                return [\n                    None if isinstance(v2, float) and v2 in _infinities else v2\n                    for v2 in row\n                ]\n    return row\n\n\nclass StaticMount(click.ParamType):\n    name = \"mount:directory\"\n\n    def convert(self, value, param, ctx):\n        if \":\" not in value:\n            self.fail(\n                f'\"{value}\" should be of format mountpoint:directory',\n                param,\n                ctx,\n            )\n        path, dirpath = value.split(\":\", 1)\n        dirpath = os.path.abspath(dirpath)\n        if not os.path.exists(dirpath) or not os.path.isdir(dirpath):\n            self.fail(f\"{value} is not a valid directory path\", param, ctx)\n        return path, dirpath\n\n\n# The --load-extension parameter can optionally include a specific entrypoint.\n# This is done by appending \":entrypoint_name\" after supplying the path to the extension\nclass LoadExtension(click.ParamType):\n    name = \"path:entrypoint?\"\n\n    def convert(self, value, param, ctx):\n        if \":\" not in value:\n            return value\n        path, entrypoint = value.split(\":\", 1)\n        return path, entrypoint\n\n\ndef format_bytes(bytes):\n    current = float(bytes)\n    for unit in (\"bytes\", \"KB\", \"MB\", \"GB\", \"TB\"):\n        if current < 1024:\n            break\n        current = current / 1024\n    if unit == \"bytes\":\n        return f\"{int(current)} {unit}\"\n    else:\n        return f\"{current:.1f} {unit}\"\n\n\n_escape_fts_re = re.compile(r'\\s+|(\".*?\")')\n\n\ndef escape_fts(query):\n    # If query has unbalanced \", add one at end\n    if query.count('\"') % 2:\n        query += '\"'\n    bits = _escape_fts_re.split(query)\n    bits = [b for b in bits if b and b != '\"\"']\n    return \" \".join(\n        '\"{}\"'.format(bit) if not bit.startswith('\"') else bit for bit in bits\n    )\n\n\nclass MultiParams:\n    def __init__(self, data):\n        # data is a dictionary of key => [list, of, values] or a list of [[\"key\", \"value\"]] pairs\n        if isinstance(data, dict):\n            for key in data:\n                assert isinstance(\n                    data[key], (list, tuple)\n                ), \"dictionary data should be a dictionary of key => [list]\"\n            self._data = data\n        elif isinstance(data, list) or isinstance(data, tuple):\n            new_data = {}\n            for item in data:\n                assert (\n                    isinstance(item, (list, tuple)) and len(item) == 2\n                ), \"list data should be a list of [key, value] pairs\"\n                key, value = item\n                new_data.setdefault(key, []).append(value)\n            self._data = new_data\n\n    def __repr__(self):\n        return f\"\"\n\n    def __contains__(self, key):\n        return key in self._data\n\n    def __getitem__(self, key):\n        return self._data[key][0]\n\n    def keys(self):\n        return self._data.keys()\n\n    def __iter__(self):\n        yield from self._data.keys()\n\n    def __len__(self):\n        return len(self._data)\n\n    def get(self, name, default=None):\n        \"\"\"Return first value in the list, if available\"\"\"\n        try:\n            return self._data.get(name)[0]\n        except (KeyError, TypeError):\n            return default\n\n    def getlist(self, name):\n        \"\"\"Return full list\"\"\"\n        return self._data.get(name) or []\n\n\nclass ConnectionProblem(Exception):\n    pass\n\n\nclass SpatialiteConnectionProblem(ConnectionProblem):\n    pass\n\n\ndef check_connection(conn):\n    tables = [\n        r[0]\n        for r in conn.execute(\n            \"select name from sqlite_master where type='table'\"\n        ).fetchall()\n    ]\n    for table in tables:\n        try:\n            conn.execute(\n                f\"PRAGMA table_info({escape_sqlite(table)});\",\n            )\n        except sqlite3.OperationalError as e:\n            if e.args[0] == \"no such module: VirtualSpatialIndex\":\n                raise SpatialiteConnectionProblem(e)\n            else:\n                raise ConnectionProblem(e)\n\n\nclass BadMetadataError(Exception):\n    pass\n\n\n@documented\ndef parse_metadata(content: str) -> dict:\n    \"Detects if content is JSON or YAML and parses it appropriately.\"\n    # content can be JSON or YAML\n    try:\n        return json.loads(content)\n    except json.JSONDecodeError:\n        try:\n            return yaml.safe_load(content)\n        except yaml.YAMLError:\n            raise BadMetadataError(\"Metadata is not valid JSON or YAML\")\n\n\ndef _gather_arguments(fn, kwargs):\n    parameters = inspect.signature(fn).parameters.keys()\n    call_with = []\n    for parameter in parameters:\n        if parameter not in kwargs:\n            raise TypeError(\n                \"{} requires parameters {}, missing: {}\".format(\n                    fn, tuple(parameters), set(parameters) - set(kwargs.keys())\n                )\n            )\n        call_with.append(kwargs[parameter])\n    return call_with\n\n\ndef call_with_supported_arguments(fn, **kwargs):\n    call_with = _gather_arguments(fn, kwargs)\n    return fn(*call_with)\n\n\nasync def async_call_with_supported_arguments(fn, **kwargs):\n    call_with = _gather_arguments(fn, kwargs)\n    return await fn(*call_with)\n\n\ndef actor_matches_allow(actor, allow):\n    if allow is True:\n        return True\n    if allow is False:\n        return False\n    if actor is None and allow and allow.get(\"unauthenticated\") is True:\n        return True\n    if allow is None:\n        return True\n    actor = actor or {}\n    for key, values in allow.items():\n        if values == \"*\" and key in actor:\n            return True\n        if not isinstance(values, list):\n            values = [values]\n        actor_values = actor.get(key)\n        if actor_values is None:\n            continue\n        if not isinstance(actor_values, list):\n            actor_values = [actor_values]\n        actor_values = set(actor_values)\n        if actor_values.intersection(values):\n            return True\n    return False\n\n\ndef resolve_env_secrets(config, environ):\n    \"\"\"Create copy that recursively replaces {\"$env\": \"NAME\"} with values from environ\"\"\"\n    if isinstance(config, dict):\n        if list(config.keys()) == [\"$env\"]:\n            return environ.get(list(config.values())[0])\n        elif list(config.keys()) == [\"$file\"]:\n            with open(list(config.values())[0]) as fp:\n                return fp.read()\n        else:\n            return {\n                key: resolve_env_secrets(value, environ)\n                for key, value in config.items()\n            }\n    elif isinstance(config, list):\n        return [resolve_env_secrets(value, environ) for value in config]\n    else:\n        return config\n\n\ndef display_actor(actor):\n    for key in (\"display\", \"name\", \"username\", \"login\", \"id\"):\n        if actor.get(key):\n            return actor[key]\n    return str(actor)\n\n\nclass SpatialiteNotFound(Exception):\n    pass\n\n\n# Can replace with sqlite-utils when I add that dependency\ndef find_spatialite():\n    for path in SPATIALITE_PATHS:\n        if os.path.exists(path):\n            return path\n    raise SpatialiteNotFound\n\n\nasync def initial_path_for_datasette(datasette):\n    \"\"\"Return suggested path for opening this Datasette, based on number of DBs and tables\"\"\"\n    databases = dict([p for p in datasette.databases.items() if p[0] != \"_internal\"])\n    if len(databases) == 1:\n        db_name = next(iter(databases.keys()))\n        path = datasette.urls.database(db_name)\n        # Does this DB only have one table?\n        db = next(iter(databases.values()))\n        tables = await db.table_names()\n        if len(tables) == 1:\n            path = datasette.urls.table(db_name, tables[0])\n    else:\n        path = datasette.urls.instance()\n    return path\n\n\nclass PrefixedUrlString(str):\n    def __add__(self, other):\n        return type(self)(super().__add__(other))\n\n    def __str__(self):\n        return super().__str__()\n\n    def __getattribute__(self, name):\n        if not name.startswith(\"__\") and name in dir(str):\n\n            def method(self, *args, **kwargs):\n                value = getattr(super(), name)(*args, **kwargs)\n                if isinstance(value, str):\n                    return type(self)(value)\n                elif isinstance(value, list):\n                    return [type(self)(i) for i in value]\n                elif isinstance(value, tuple):\n                    return tuple(type(self)(i) for i in value)\n                else:\n                    return value\n\n            return method.__get__(self)\n        else:\n            return super().__getattribute__(name)\n\n\nclass StartupError(Exception):\n    pass\n\n\n_single_line_comment_re = re.compile(r\"--.*\")\n_multi_line_comment_re = re.compile(r\"/\\*.*?\\*/\", re.DOTALL)\n_single_quote_re = re.compile(r\"'(?:''|[^'])*'\")\n_double_quote_re = re.compile(r'\"(?:\\\"\\\"|[^\"])*\"')\n_named_param_re = re.compile(r\":(\\w+)\")\n\n\n@documented\ndef named_parameters(sql: str) -> List[str]:\n    \"\"\"\n    Given a SQL statement, return a list of named parameters that are used in the statement\n\n    e.g. for ``select * from foo where id=:id`` this would return ``[\"id\"]``\n    \"\"\"\n    sql = _single_line_comment_re.sub(\"\", sql)\n    sql = _multi_line_comment_re.sub(\"\", sql)\n    sql = _single_quote_re.sub(\"\", sql)\n    sql = _double_quote_re.sub(\"\", sql)\n    # Extract parameters from what is left\n    return _named_param_re.findall(sql)\n\n\nasync def derive_named_parameters(db: \"Database\", sql: str) -> List[str]:\n    \"\"\"\n    This undocumented but stable method exists for backwards compatibility\n    with plugins that were using it before it switched to named_parameters()\n    \"\"\"\n    return named_parameters(sql)\n\n\ndef add_cors_headers(headers):\n    headers[\"Access-Control-Allow-Origin\"] = \"*\"\n    headers[\"Access-Control-Allow-Headers\"] = \"Authorization, Content-Type\"\n    headers[\"Access-Control-Expose-Headers\"] = \"Link\"\n    headers[\"Access-Control-Allow-Methods\"] = \"GET, POST, HEAD, OPTIONS\"\n    headers[\"Access-Control-Max-Age\"] = \"3600\"\n\n\n_TILDE_ENCODING_SAFE = frozenset(\n    b\"ABCDEFGHIJKLMNOPQRSTUVWXYZ\"\n    b\"abcdefghijklmnopqrstuvwxyz\"\n    b\"0123456789_-\"\n    # This is the same as Python percent-encoding but I removed\n    # '.' and '~'\n)\n\n_space = ord(\" \")\n\n\nclass TildeEncoder(dict):\n    # Keeps a cache internally, via __missing__\n    def __missing__(self, b):\n        # Handle a cache miss, store encoded string in cache and return.\n        if b in _TILDE_ENCODING_SAFE:\n            res = chr(b)\n        elif b == _space:\n            res = \"+\"\n        else:\n            res = \"~{:02X}\".format(b)\n        self[b] = res\n        return res\n\n\n_tilde_encoder = TildeEncoder().__getitem__\n\n\n@documented\ndef tilde_encode(s: str) -> str:\n    \"Returns tilde-encoded string - for example ``/foo/bar`` -> ``~2Ffoo~2Fbar``\"\n    return \"\".join(_tilde_encoder(char) for char in s.encode(\"utf-8\"))\n\n\n@documented\ndef tilde_decode(s: str) -> str:\n    \"Decodes a tilde-encoded string, so ``~2Ffoo~2Fbar`` -> ``/foo/bar``\"\n    # Avoid accidentally decoding a %2f style sequence\n    temp = secrets.token_hex(16)\n    s = s.replace(\"%\", temp)\n    decoded = urllib.parse.unquote_plus(s.replace(\"~\", \"%\"))\n    return decoded.replace(temp, \"%\")\n\n\ndef resolve_routes(routes, path):\n    for regex, view in routes:\n        match = regex.match(path)\n        if match is not None:\n            return match, view\n    return None, None\n\n\ndef truncate_url(url, length):\n    if (not length) or (len(url) <= length):\n        return url\n    bits = url.rsplit(\".\", 1)\n    if len(bits) == 2 and 1 <= len(bits[1]) <= 4 and \"/\" not in bits[1]:\n        rest, ext = bits\n        return rest[: length - 1 - len(ext)] + \"….\" + ext\n    return url[: length - 1] + \"…\"\n\n\nasync def row_sql_params_pks(db, table, pk_values):\n    pks = await db.primary_keys(table)\n    use_rowid = not pks\n    select = \"*\"\n    if use_rowid:\n        select = \"rowid, *\"\n        pks = [\"rowid\"]\n    wheres = [f'\"{pk}\"=:p{i}' for i, pk in enumerate(pks)]\n    sql = f\"select {select} from {escape_sqlite(table)} where {' AND '.join(wheres)}\"\n    params = {}\n    for i, pk_value in enumerate(pk_values):\n        params[f\"p{i}\"] = pk_value\n    return sql, params, pks\n\n\ndef _handle_pair(key: str, value: str) -> dict:\n    \"\"\"\n    Turn a key-value pair into a nested dictionary.\n    foo, bar => {'foo': 'bar'}\n    foo.bar, baz => {'foo': {'bar': 'baz'}}\n    foo.bar, [1, 2, 3] => {'foo': {'bar': [1, 2, 3]}}\n    foo.bar, \"baz\" => {'foo': {'bar': 'baz'}}\n    foo.bar, '{\"baz\": \"qux\"}' => {'foo': {'bar': \"{'baz': 'qux'}\"}}\n    \"\"\"\n    try:\n        value = json.loads(value)\n    except json.JSONDecodeError:\n        # If it doesn't parse as JSON, treat it as a string\n        pass\n\n    keys = key.split(\".\")\n    result = current_dict = {}\n\n    for k in keys[:-1]:\n        current_dict[k] = {}\n        current_dict = current_dict[k]\n\n    current_dict[keys[-1]] = value\n    return result\n\n\ndef _combine(base: dict, update: dict) -> dict:\n    \"\"\"\n    Recursively merge two dictionaries.\n    \"\"\"\n    for key, value in update.items():\n        if isinstance(value, dict) and key in base and isinstance(base[key], dict):\n            base[key] = _combine(base[key], value)\n        else:\n            base[key] = value\n    return base\n\n\ndef pairs_to_nested_config(pairs: typing.List[typing.Tuple[str, typing.Any]]) -> dict:\n    \"\"\"\n    Parse a list of key-value pairs into a nested dictionary.\n    \"\"\"\n    result = {}\n    for key, value in pairs:\n        parsed_pair = _handle_pair(key, value)\n        result = _combine(result, parsed_pair)\n    return result\n\n\ndef make_slot_function(name, datasette, request, **kwargs):\n    from datasette.plugins import pm\n\n    method = getattr(pm.hook, name, None)\n    assert method is not None, \"No hook found for {}\".format(name)\n\n    async def inner():\n        html_bits = []\n        for hook in method(datasette=datasette, request=request, **kwargs):\n            html = await await_me_maybe(hook)\n            if html is not None:\n                html_bits.append(html)\n        return markupsafe.Markup(\"\".join(html_bits))\n\n    return inner\n\n\ndef prune_empty_dicts(d: dict):\n    \"\"\"\n    Recursively prune all empty dictionaries from a given dictionary.\n    \"\"\"\n    for key, value in list(d.items()):\n        if isinstance(value, dict):\n            prune_empty_dicts(value)\n            if value == {}:\n                d.pop(key, None)\n\n\ndef move_plugins_and_allow(source: dict, destination: dict) -> Tuple[dict, dict]:\n    \"\"\"\n    Move 'plugins' and 'allow' keys from source to destination dictionary. Creates\n    hierarchy in destination if needed. After moving, recursively remove any keys\n    in the source that are left empty.\n    \"\"\"\n    source = copy.deepcopy(source)\n    destination = copy.deepcopy(destination)\n\n    def recursive_move(src, dest, path=None):\n        if path is None:\n            path = []\n        for key, value in list(src.items()):\n            new_path = path + [key]\n            if key in (\"plugins\", \"allow\"):\n                # Navigate and create the hierarchy in destination if needed\n                d = dest\n                for step in path:\n                    d = d.setdefault(step, {})\n                # Move the plugins\n                d[key] = value\n                # Remove the plugins from source\n                src.pop(key, None)\n            elif isinstance(value, dict):\n                recursive_move(value, dest, new_path)\n                # After moving, check if the current dictionary is empty and remove it if so\n                if not value:\n                    src.pop(key, None)\n\n    recursive_move(source, destination)\n    prune_empty_dicts(source)\n    return source, destination\n\n\n_table_config_keys = (\n    \"hidden\",\n    \"sort\",\n    \"sort_desc\",\n    \"size\",\n    \"sortable_columns\",\n    \"label_column\",\n    \"facets\",\n    \"fts_table\",\n    \"fts_pk\",\n    \"searchmode\",\n)\n\n\ndef move_table_config(metadata: dict, config: dict):\n    \"\"\"\n    Move all known table configuration keys from metadata to config.\n    \"\"\"\n    if \"databases\" not in metadata:\n        return metadata, config\n    metadata = copy.deepcopy(metadata)\n    config = copy.deepcopy(config)\n    for database_name, database in metadata[\"databases\"].items():\n        if \"tables\" not in database:\n            continue\n        for table_name, table in database[\"tables\"].items():\n            for key in _table_config_keys:\n                if key in table:\n                    config.setdefault(\"databases\", {}).setdefault(\n                        database_name, {}\n                    ).setdefault(\"tables\", {}).setdefault(table_name, {})[\n                        key\n                    ] = table.pop(\n                        key\n                    )\n    prune_empty_dicts(metadata)\n    return metadata, config\n\n\ndef redact_keys(original: dict, key_patterns: Iterable) -> dict:\n    \"\"\"\n    Recursively redact sensitive keys in a dictionary based on given patterns\n\n    :param original: The original dictionary\n    :param key_patterns: A list of substring patterns to redact\n    :return: A copy of the original dictionary with sensitive values redacted\n    \"\"\"\n\n    def redact(data):\n        if isinstance(data, dict):\n            return {\n                k: (\n                    redact(v)\n                    if not any(pattern in k for pattern in key_patterns)\n                    else \"***\"\n                )\n                for k, v in data.items()\n            }\n        elif isinstance(data, list):\n            return [redact(item) for item in data]\n        else:\n            return data\n\n    return redact(original)\n\n\ndef md5_not_usedforsecurity(s):\n    try:\n        return hashlib.md5(s.encode(\"utf8\"), usedforsecurity=False).hexdigest()\n    except TypeError:\n        # For Python 3.8 which does not support usedforsecurity=False\n        return hashlib.md5(s.encode(\"utf8\")).hexdigest()\n\n\n_etag_cache = {}\n\n\nasync def calculate_etag(filepath, chunk_size=4096):\n    if filepath in _etag_cache:\n        return _etag_cache[filepath]\n\n    hasher = hashlib.md5()\n    async with aiofiles.open(filepath, \"rb\") as f:\n        while True:\n            chunk = await f.read(chunk_size)\n            if not chunk:\n                break\n            hasher.update(chunk)\n\n    etag = f'\"{hasher.hexdigest()}\"'\n    _etag_cache[filepath] = etag\n\n    return etag\n\n\ndef deep_dict_update(dict1, dict2):\n    for key, value in dict2.items():\n        if isinstance(value, dict):\n            dict1[key] = deep_dict_update(dict1.get(key, type(value)()), value)\n        else:\n            dict1[key] = value\n    return dict1\n"
  },
  {
    "path": "datasette/utils/actions_sql.py",
    "content": "\"\"\"\nSQL query builder for hierarchical permission checking.\n\nThis module implements a cascading permission system based on the pattern\nfrom https://github.com/simonw/research/tree/main/sqlite-permissions-poc\n\nIt builds SQL queries that:\n\n1. Start with all resources of a given type (from resource_type.resources_sql())\n2. Gather permission rules from plugins (via permission_resources_sql hook)\n3. Apply cascading logic: child → parent → global\n4. Apply DENY-beats-ALLOW at each level\n\nThe core pattern is:\n- Resources are identified by (parent, child) tuples\n- Rules are evaluated at three levels:\n  - child: exact match on (parent, child)\n  - parent: match on (parent, NULL)\n  - global: match on (NULL, NULL)\n- At the same level, DENY (allow=0) beats ALLOW (allow=1)\n- Across levels, child beats parent beats global\n\"\"\"\n\nfrom typing import TYPE_CHECKING\n\nfrom datasette.utils.permissions import gather_permission_sql_from_hooks\n\nif TYPE_CHECKING:\n    from datasette.app import Datasette\n\n\nasync def build_allowed_resources_sql(\n    datasette: \"Datasette\",\n    actor: dict | None,\n    action: str,\n    *,\n    parent: str | None = None,\n    include_is_private: bool = False,\n) -> tuple[str, dict]:\n    \"\"\"\n    Build a SQL query that returns all resources the actor can access for this action.\n\n    Args:\n        datasette: The Datasette instance\n        actor: The actor dict (or None for unauthenticated)\n        action: The action name (e.g., \"view-table\", \"view-database\")\n        parent: Optional parent filter to limit results (e.g., database name)\n        include_is_private: If True, add is_private column showing if anonymous cannot access\n\n    Returns:\n        A tuple of (sql_query, params_dict)\n\n    The returned SQL query will have three columns (or four with include_is_private):\n        - parent: The parent resource identifier (or NULL)\n        - child: The child resource identifier (or NULL)\n        - reason: The reason from the rule that granted access\n        - is_private: (if include_is_private) 1 if anonymous cannot access, 0 otherwise\n\n    Example:\n        For action=\"view-table\", this might return:\n        SELECT parent, child, reason FROM ... WHERE is_allowed = 1\n\n        Results would be like:\n        ('analytics', 'users', 'role-based: analysts can access analytics DB')\n        ('analytics', 'events', 'role-based: analysts can access analytics DB')\n        ('production', 'orders', 'business-exception: allow production.orders for carol')\n    \"\"\"\n    # Get the Action object\n    action_obj = datasette.actions.get(action)\n    if not action_obj:\n        raise ValueError(f\"Unknown action: {action}\")\n\n    # If this action also_requires another action, we need to combine the queries\n    if action_obj.also_requires:\n        # Build both queries\n        main_sql, main_params = await _build_single_action_sql(\n            datasette,\n            actor,\n            action,\n            parent=parent,\n            include_is_private=include_is_private,\n        )\n        required_sql, required_params = await _build_single_action_sql(\n            datasette,\n            actor,\n            action_obj.also_requires,\n            parent=parent,\n            include_is_private=False,\n        )\n\n        # Merge parameters - they should have identical values for :actor, :actor_id, etc.\n        all_params = {**main_params, **required_params}\n        if parent is not None:\n            all_params[\"filter_parent\"] = parent\n\n        # Combine with INNER JOIN - only resources allowed by both actions\n        combined_sql = f\"\"\"\nWITH\nmain_allowed AS (\n{main_sql}\n),\nrequired_allowed AS (\n{required_sql}\n)\nSELECT m.parent, m.child, m.reason\"\"\"\n\n        if include_is_private:\n            combined_sql += \", m.is_private\"\n\n        combined_sql += \"\"\"\nFROM main_allowed m\nINNER JOIN required_allowed r\n  ON ((m.parent = r.parent) OR (m.parent IS NULL AND r.parent IS NULL))\n AND ((m.child = r.child) OR (m.child IS NULL AND r.child IS NULL))\n\"\"\"\n\n        if parent is not None:\n            combined_sql += \"WHERE m.parent = :filter_parent\\n\"\n\n        combined_sql += \"ORDER BY m.parent, m.child\"\n\n        return combined_sql, all_params\n\n    # No also_requires, build single action query\n    return await _build_single_action_sql(\n        datasette, actor, action, parent=parent, include_is_private=include_is_private\n    )\n\n\nasync def _build_single_action_sql(\n    datasette: \"Datasette\",\n    actor: dict | None,\n    action: str,\n    *,\n    parent: str | None = None,\n    include_is_private: bool = False,\n) -> tuple[str, dict]:\n    \"\"\"\n    Build SQL for a single action (internal helper for build_allowed_resources_sql).\n\n    This contains the original logic from build_allowed_resources_sql, extracted\n    to allow combining multiple actions when also_requires is used.\n    \"\"\"\n    # Get the Action object\n    action_obj = datasette.actions.get(action)\n    if not action_obj:\n        raise ValueError(f\"Unknown action: {action}\")\n\n    # Get base resources SQL from the resource class\n    base_resources_sql = await action_obj.resource_class.resources_sql(\n        datasette, actor=actor\n    )\n\n    permission_sqls = await gather_permission_sql_from_hooks(\n        datasette=datasette,\n        actor=actor,\n        action=action,\n    )\n\n    # If permission_sqls is the sentinel, skip all permission checks\n    # Return SQL that allows all resources\n    from datasette.utils.permissions import SKIP_PERMISSION_CHECKS\n\n    if permission_sqls is SKIP_PERMISSION_CHECKS:\n        cols = \"parent, child, 'skip_permission_checks' AS reason\"\n        if include_is_private:\n            cols += \", 0 AS is_private\"\n        return f\"SELECT {cols} FROM ({base_resources_sql})\", {}\n\n    all_params = {}\n    rule_sqls = []\n    restriction_sqls = []\n\n    for permission_sql in permission_sqls:\n        # Always collect params (even from restriction-only plugins)\n        all_params.update(permission_sql.params or {})\n\n        # Collect restriction SQL filters\n        if permission_sql.restriction_sql:\n            restriction_sqls.append(permission_sql.restriction_sql)\n\n        # Skip plugins that only provide restriction_sql (no permission rules)\n        if permission_sql.sql is None:\n            continue\n        rule_sqls.append(f\"\"\"\n            SELECT parent, child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (\n                {permission_sql.sql}\n            )\n            \"\"\".strip())\n\n    # If no rules, return empty result (deny all)\n    if not rule_sqls:\n        empty_cols = \"NULL AS parent, NULL AS child, NULL AS reason\"\n        if include_is_private:\n            empty_cols += \", NULL AS is_private\"\n        return f\"SELECT {empty_cols} WHERE 0\", {}\n\n    # Build the cascading permission query\n    rules_union = \" UNION ALL \".join(rule_sqls)\n\n    # Build the main query\n    query_parts = [\n        \"WITH\",\n        \"base AS (\",\n        f\"  {base_resources_sql}\",\n        \"),\",\n        \"all_rules AS (\",\n        f\"  {rules_union}\",\n        \"),\",\n    ]\n\n    # If include_is_private, we need to build anonymous permissions too\n    if include_is_private:\n        anon_permission_sqls = await gather_permission_sql_from_hooks(\n            datasette=datasette,\n            actor=None,\n            action=action,\n        )\n        anon_sqls_rewritten = []\n        anon_params = {}\n\n        for permission_sql in anon_permission_sqls:\n            # Skip plugins that only provide restriction_sql (no permission rules)\n            if permission_sql.sql is None:\n                continue\n            rewritten_sql = permission_sql.sql\n            for key, value in (permission_sql.params or {}).items():\n                anon_key = f\"anon_{key}\"\n                anon_params[anon_key] = value\n                rewritten_sql = rewritten_sql.replace(f\":{key}\", f\":{anon_key}\")\n            anon_sqls_rewritten.append(rewritten_sql)\n\n        all_params.update(anon_params)\n\n        if anon_sqls_rewritten:\n            anon_rules_union = \" UNION ALL \".join(anon_sqls_rewritten)\n            query_parts.extend(\n                [\n                    \"anon_rules AS (\",\n                    f\"  {anon_rules_union}\",\n                    \"),\",\n                ]\n            )\n\n    # Continue with the cascading logic\n    query_parts.extend(\n        [\n            \"child_lvl AS (\",\n            \"  SELECT b.parent, b.child,\",\n            \"         MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,\",\n            \"         MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow,\",\n            \"         json_group_array(CASE WHEN ar.allow = 0 THEN ar.source_plugin || ': ' || ar.reason END) AS deny_reasons,\",\n            \"         json_group_array(CASE WHEN ar.allow = 1 THEN ar.source_plugin || ': ' || ar.reason END) AS allow_reasons\",\n            \"  FROM base b\",\n            \"  LEFT JOIN all_rules ar ON ar.parent = b.parent AND ar.child = b.child\",\n            \"  GROUP BY b.parent, b.child\",\n            \"),\",\n            \"parent_lvl AS (\",\n            \"  SELECT b.parent, b.child,\",\n            \"         MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,\",\n            \"         MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow,\",\n            \"         json_group_array(CASE WHEN ar.allow = 0 THEN ar.source_plugin || ': ' || ar.reason END) AS deny_reasons,\",\n            \"         json_group_array(CASE WHEN ar.allow = 1 THEN ar.source_plugin || ': ' || ar.reason END) AS allow_reasons\",\n            \"  FROM base b\",\n            \"  LEFT JOIN all_rules ar ON ar.parent = b.parent AND ar.child IS NULL\",\n            \"  GROUP BY b.parent, b.child\",\n            \"),\",\n            \"global_lvl AS (\",\n            \"  SELECT b.parent, b.child,\",\n            \"         MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,\",\n            \"         MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow,\",\n            \"         json_group_array(CASE WHEN ar.allow = 0 THEN ar.source_plugin || ': ' || ar.reason END) AS deny_reasons,\",\n            \"         json_group_array(CASE WHEN ar.allow = 1 THEN ar.source_plugin || ': ' || ar.reason END) AS allow_reasons\",\n            \"  FROM base b\",\n            \"  LEFT JOIN all_rules ar ON ar.parent IS NULL AND ar.child IS NULL\",\n            \"  GROUP BY b.parent, b.child\",\n            \"),\",\n        ]\n    )\n\n    # Add anonymous decision logic if needed\n    if include_is_private:\n        query_parts.extend(\n            [\n                \"anon_child_lvl AS (\",\n                \"  SELECT b.parent, b.child,\",\n                \"         MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,\",\n                \"         MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow\",\n                \"  FROM base b\",\n                \"  LEFT JOIN anon_rules ar ON ar.parent = b.parent AND ar.child = b.child\",\n                \"  GROUP BY b.parent, b.child\",\n                \"),\",\n                \"anon_parent_lvl AS (\",\n                \"  SELECT b.parent, b.child,\",\n                \"         MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,\",\n                \"         MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow\",\n                \"  FROM base b\",\n                \"  LEFT JOIN anon_rules ar ON ar.parent = b.parent AND ar.child IS NULL\",\n                \"  GROUP BY b.parent, b.child\",\n                \"),\",\n                \"anon_global_lvl AS (\",\n                \"  SELECT b.parent, b.child,\",\n                \"         MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,\",\n                \"         MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow\",\n                \"  FROM base b\",\n                \"  LEFT JOIN anon_rules ar ON ar.parent IS NULL AND ar.child IS NULL\",\n                \"  GROUP BY b.parent, b.child\",\n                \"),\",\n                \"anon_decisions AS (\",\n                \"  SELECT\",\n                \"    b.parent, b.child,\",\n                \"    CASE\",\n                \"      WHEN acl.any_deny = 1 THEN 0\",\n                \"      WHEN acl.any_allow = 1 THEN 1\",\n                \"      WHEN apl.any_deny = 1 THEN 0\",\n                \"      WHEN apl.any_allow = 1 THEN 1\",\n                \"      WHEN agl.any_deny = 1 THEN 0\",\n                \"      WHEN agl.any_allow = 1 THEN 1\",\n                \"      ELSE 0\",\n                \"    END AS anon_is_allowed\",\n                \"  FROM base b\",\n                \"  JOIN anon_child_lvl acl ON b.parent = acl.parent AND (b.child = acl.child OR (b.child IS NULL AND acl.child IS NULL))\",\n                \"  JOIN anon_parent_lvl apl ON b.parent = apl.parent AND (b.child = apl.child OR (b.child IS NULL AND apl.child IS NULL))\",\n                \"  JOIN anon_global_lvl agl ON b.parent = agl.parent AND (b.child = agl.child OR (b.child IS NULL AND agl.child IS NULL))\",\n                \"),\",\n            ]\n        )\n\n    # Final decisions\n    query_parts.extend(\n        [\n            \"decisions AS (\",\n            \"  SELECT\",\n            \"    b.parent, b.child,\",\n            \"    -- Cascading permission logic: child → parent → global, DENY beats ALLOW at each level\",\n            \"    -- Priority order:\",\n            \"    --   1. Child-level deny (most specific, blocks access)\",\n            \"    --   2. Child-level allow (most specific, grants access)\",\n            \"    --   3. Parent-level deny (intermediate, blocks access)\",\n            \"    --   4. Parent-level allow (intermediate, grants access)\",\n            \"    --   5. Global-level deny (least specific, blocks access)\",\n            \"    --   6. Global-level allow (least specific, grants access)\",\n            \"    --   7. Default deny (no rules match)\",\n            \"    CASE\",\n            \"      WHEN cl.any_deny = 1 THEN 0\",\n            \"      WHEN cl.any_allow = 1 THEN 1\",\n            \"      WHEN pl.any_deny = 1 THEN 0\",\n            \"      WHEN pl.any_allow = 1 THEN 1\",\n            \"      WHEN gl.any_deny = 1 THEN 0\",\n            \"      WHEN gl.any_allow = 1 THEN 1\",\n            \"      ELSE 0\",\n            \"    END AS is_allowed,\",\n            \"    CASE\",\n            \"      WHEN cl.any_deny = 1 THEN cl.deny_reasons\",\n            \"      WHEN cl.any_allow = 1 THEN cl.allow_reasons\",\n            \"      WHEN pl.any_deny = 1 THEN pl.deny_reasons\",\n            \"      WHEN pl.any_allow = 1 THEN pl.allow_reasons\",\n            \"      WHEN gl.any_deny = 1 THEN gl.deny_reasons\",\n            \"      WHEN gl.any_allow = 1 THEN gl.allow_reasons\",\n            \"      ELSE '[]'\",\n            \"    END AS reason\",\n        ]\n    )\n\n    if include_is_private:\n        query_parts.append(\n            \"    , CASE WHEN ad.anon_is_allowed = 0 THEN 1 ELSE 0 END AS is_private\"\n        )\n\n    query_parts.extend(\n        [\n            \"  FROM base b\",\n            \"  JOIN child_lvl cl ON b.parent = cl.parent AND (b.child = cl.child OR (b.child IS NULL AND cl.child IS NULL))\",\n            \"  JOIN parent_lvl pl ON b.parent = pl.parent AND (b.child = pl.child OR (b.child IS NULL AND pl.child IS NULL))\",\n            \"  JOIN global_lvl gl ON b.parent = gl.parent AND (b.child = gl.child OR (b.child IS NULL AND gl.child IS NULL))\",\n        ]\n    )\n\n    if include_is_private:\n        query_parts.append(\n            \"  JOIN anon_decisions ad ON b.parent = ad.parent AND (b.child = ad.child OR (b.child IS NULL AND ad.child IS NULL))\"\n        )\n\n    query_parts.append(\")\")\n\n    # Add restriction list CTE if there are restrictions\n    if restriction_sqls:\n        # Wrap each restriction_sql in a subquery to avoid operator precedence issues\n        # with UNION ALL inside the restriction SQL statements\n        restriction_intersect = \"\\nINTERSECT\\n\".join(\n            f\"SELECT * FROM ({sql})\" for sql in restriction_sqls\n        )\n        query_parts.extend(\n            [\",\", \"restriction_list AS (\", f\"  {restriction_intersect}\", \")\"]\n        )\n\n    # Final SELECT\n    select_cols = \"parent, child, reason\"\n    if include_is_private:\n        select_cols += \", is_private\"\n\n    query_parts.append(f\"SELECT {select_cols}\")\n    query_parts.append(\"FROM decisions\")\n    query_parts.append(\"WHERE is_allowed = 1\")\n\n    # Add restriction filter if there are restrictions\n    if restriction_sqls:\n        query_parts.append(\"\"\"\n  AND EXISTS (\n    SELECT 1 FROM restriction_list r\n    WHERE (r.parent = decisions.parent OR r.parent IS NULL)\n      AND (r.child = decisions.child OR r.child IS NULL)\n  )\"\"\")\n\n    # Add parent filter if specified\n    if parent is not None:\n        query_parts.append(\"  AND parent = :filter_parent\")\n        all_params[\"filter_parent\"] = parent\n\n    query_parts.append(\"ORDER BY parent, child\")\n\n    query = \"\\n\".join(query_parts)\n    return query, all_params\n\n\nasync def build_permission_rules_sql(\n    datasette: \"Datasette\", actor: dict | None, action: str\n) -> tuple[str, dict]:\n    \"\"\"\n    Build the UNION SQL and params for all permission rules for a given actor and action.\n\n    Returns:\n        A tuple of (sql, params) where sql is a UNION ALL query that returns\n        (parent, child, allow, reason, source_plugin) rows.\n    \"\"\"\n    # Get the Action object\n    action_obj = datasette.actions.get(action)\n    if not action_obj:\n        raise ValueError(f\"Unknown action: {action}\")\n\n    permission_sqls = await gather_permission_sql_from_hooks(\n        datasette=datasette,\n        actor=actor,\n        action=action,\n    )\n\n    # If permission_sqls is the sentinel, skip all permission checks\n    # Return SQL that allows everything\n    from datasette.utils.permissions import SKIP_PERMISSION_CHECKS\n\n    if permission_sqls is SKIP_PERMISSION_CHECKS:\n        return (\n            \"SELECT NULL AS parent, NULL AS child, 1 AS allow, 'skip_permission_checks' AS reason, 'skip' AS source_plugin\",\n            {},\n            [],\n        )\n\n    if not permission_sqls:\n        return (\n            \"SELECT NULL AS parent, NULL AS child, 0 AS allow, NULL AS reason, NULL AS source_plugin WHERE 0\",\n            {},\n            [],\n        )\n\n    union_parts = []\n    all_params = {}\n    restriction_sqls = []\n\n    for permission_sql in permission_sqls:\n        all_params.update(permission_sql.params or {})\n\n        # Collect restriction SQL filters\n        if permission_sql.restriction_sql:\n            restriction_sqls.append(permission_sql.restriction_sql)\n\n        # Skip plugins that only provide restriction_sql (no permission rules)\n        if permission_sql.sql is None:\n            continue\n\n        union_parts.append(f\"\"\"\n            SELECT parent, child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (\n                {permission_sql.sql}\n            )\n            \"\"\".strip())\n\n    rules_union = \" UNION ALL \".join(union_parts)\n    return rules_union, all_params, restriction_sqls\n\n\nasync def check_permission_for_resource(\n    *,\n    datasette: \"Datasette\",\n    actor: dict | None,\n    action: str,\n    parent: str | None,\n    child: str | None,\n) -> bool:\n    \"\"\"\n    Check if an actor has permission for a specific action on a specific resource.\n\n    Args:\n        datasette: The Datasette instance\n        actor: The actor dict (or None)\n        action: The action name\n        parent: The parent resource identifier (e.g., database name, or None)\n        child: The child resource identifier (e.g., table name, or None)\n\n    Returns:\n        True if the actor is allowed, False otherwise\n\n    This builds the cascading permission query and checks if the specific\n    resource is in the allowed set.\n    \"\"\"\n    rules_union, all_params, restriction_sqls = await build_permission_rules_sql(\n        datasette, actor, action\n    )\n\n    # If no rules (empty SQL), default deny\n    if not rules_union:\n        return False\n\n    # Add parameters for the resource we're checking\n    all_params[\"_check_parent\"] = parent\n    all_params[\"_check_child\"] = child\n\n    # If there are restriction filters, check if the resource passes them first\n    if restriction_sqls:\n        # Check if resource is in restriction allowlist\n        # Database-level restrictions (parent, NULL) should match all children (parent, *)\n        # Wrap each restriction_sql in a subquery to avoid operator precedence issues\n        restriction_check = \"\\nINTERSECT\\n\".join(\n            f\"SELECT * FROM ({sql})\" for sql in restriction_sqls\n        )\n        restriction_query = f\"\"\"\nWITH restriction_list AS (\n    {restriction_check}\n)\nSELECT EXISTS (\n    SELECT 1 FROM restriction_list\n    WHERE (parent = :_check_parent OR parent IS NULL)\n      AND (child = :_check_child OR child IS NULL)\n) AS in_allowlist\n\"\"\"\n        result = await datasette.get_internal_database().execute(\n            restriction_query, all_params\n        )\n        if result.rows and not result.rows[0][0]:\n            # Resource not in restriction allowlist - deny\n            return False\n\n    query = f\"\"\"\nWITH\nall_rules AS (\n  {rules_union}\n),\nmatched_rules AS (\n  SELECT ar.*,\n    CASE\n      WHEN ar.child IS NOT NULL THEN 2  -- child-level (most specific)\n      WHEN ar.parent IS NOT NULL THEN 1  -- parent-level\n      ELSE 0                             -- root/global\n    END AS depth\n  FROM all_rules ar\n  WHERE (ar.parent IS NULL OR ar.parent = :_check_parent)\n    AND (ar.child IS NULL OR ar.child = :_check_child)\n),\nwinner AS (\n  SELECT *\n  FROM matched_rules\n  ORDER BY\n    depth DESC,                          -- specificity first (higher depth wins)\n    CASE WHEN allow=0 THEN 0 ELSE 1 END, -- then deny over allow\n    source_plugin                        -- stable tie-break\n  LIMIT 1\n)\nSELECT COALESCE((SELECT allow FROM winner), 0) AS is_allowed\n\"\"\"\n\n    # Execute the query against the internal database\n    result = await datasette.get_internal_database().execute(query, all_params)\n    if result.rows:\n        return bool(result.rows[0][0])\n    return False\n"
  },
  {
    "path": "datasette/utils/asgi.py",
    "content": "import json\nfrom typing import Optional\nfrom datasette.utils import MultiParams, calculate_etag\nfrom datasette.utils.multipart import (\n    parse_form_data,\n    MultipartParseError,\n    FormData,\n    DEFAULT_MAX_FILE_SIZE,\n    DEFAULT_MAX_REQUEST_SIZE,\n    DEFAULT_MAX_FIELDS,\n    DEFAULT_MAX_FILES,\n    DEFAULT_MAX_PARTS,\n    DEFAULT_MAX_FIELD_SIZE,\n    DEFAULT_MAX_MEMORY_FILE_SIZE,\n    DEFAULT_MAX_PART_HEADER_BYTES,\n    DEFAULT_MAX_PART_HEADER_LINES,\n    DEFAULT_MIN_FREE_DISK_BYTES,\n)\nfrom mimetypes import guess_type\nfrom urllib.parse import parse_qs, urlunparse, parse_qsl\nfrom pathlib import Path\nfrom http.cookies import SimpleCookie, Morsel\nimport aiofiles\nimport aiofiles.os\nimport re\n\n# Workaround for adding samesite support to pre 3.8 python\nMorsel._reserved[\"samesite\"] = \"SameSite\"\n# Thanks, Starlette:\n# https://github.com/encode/starlette/blob/519f575/starlette/responses.py#L17\n\n\nclass Base400(Exception):\n    status = 400\n\n\nclass NotFound(Base400):\n    status = 404\n\n\nclass DatabaseNotFound(NotFound):\n    def __init__(self, database_name):\n        self.database_name = database_name\n        super().__init__(\"Database not found\")\n\n\nclass TableNotFound(NotFound):\n    def __init__(self, database_name, table):\n        super().__init__(\"Table not found\")\n        self.database_name = database_name\n        self.table = table\n\n\nclass RowNotFound(NotFound):\n    def __init__(self, database_name, table, pk_values):\n        super().__init__(\"Row not found\")\n        self.database_name = database_name\n        self.table_name = table\n        self.pk_values = pk_values\n\n\nclass Forbidden(Base400):\n    status = 403\n\n\nclass BadRequest(Base400):\n    status = 400\n\n\nSAMESITE_VALUES = (\"strict\", \"lax\", \"none\")\n\n\nclass Request:\n    def __init__(self, scope, receive):\n        self.scope = scope\n        self.receive = receive\n\n    def __repr__(self):\n        return ''.format(self.method, self.url)\n\n    @property\n    def method(self):\n        return self.scope[\"method\"]\n\n    @property\n    def url(self):\n        return urlunparse(\n            (self.scheme, self.host, self.path, None, self.query_string, None)\n        )\n\n    @property\n    def url_vars(self):\n        return (self.scope.get(\"url_route\") or {}).get(\"kwargs\") or {}\n\n    @property\n    def scheme(self):\n        return self.scope.get(\"scheme\") or \"http\"\n\n    @property\n    def headers(self):\n        return {\n            k.decode(\"latin-1\").lower(): v.decode(\"latin-1\")\n            for k, v in self.scope.get(\"headers\") or []\n        }\n\n    @property\n    def host(self):\n        return self.headers.get(\"host\") or \"localhost\"\n\n    @property\n    def cookies(self):\n        cookies = SimpleCookie()\n        cookies.load(self.headers.get(\"cookie\", \"\"))\n        return {key: value.value for key, value in cookies.items()}\n\n    @property\n    def path(self):\n        if self.scope.get(\"raw_path\") is not None:\n            return self.scope[\"raw_path\"].decode(\"latin-1\").partition(\"?\")[0]\n        else:\n            path = self.scope[\"path\"]\n            if isinstance(path, str):\n                return path\n            else:\n                return path.decode(\"utf-8\")\n\n    @property\n    def query_string(self):\n        return (self.scope.get(\"query_string\") or b\"\").decode(\"latin-1\")\n\n    @property\n    def full_path(self):\n        qs = self.query_string\n        return \"{}{}\".format(self.path, (\"?\" + qs) if qs else \"\")\n\n    @property\n    def args(self):\n        return MultiParams(parse_qs(qs=self.query_string, keep_blank_values=True))\n\n    @property\n    def actor(self):\n        return self.scope.get(\"actor\", None)\n\n    async def post_body(self):\n        body = b\"\"\n        more_body = True\n        while more_body:\n            message = await self.receive()\n            assert message[\"type\"] == \"http.request\", message\n            body += message.get(\"body\", b\"\")\n            more_body = message.get(\"more_body\", False)\n        return body\n\n    async def post_vars(self):\n        body = await self.post_body()\n        return dict(parse_qsl(body.decode(\"utf-8\"), keep_blank_values=True))\n\n    async def form(\n        self,\n        files: bool = False,\n        max_file_size: int = DEFAULT_MAX_FILE_SIZE,\n        max_request_size: int = DEFAULT_MAX_REQUEST_SIZE,\n        max_fields: int = DEFAULT_MAX_FIELDS,\n        max_files: int = DEFAULT_MAX_FILES,\n        max_parts: Optional[int] = DEFAULT_MAX_PARTS,\n        max_field_size: int = DEFAULT_MAX_FIELD_SIZE,\n        max_memory_file_size: int = DEFAULT_MAX_MEMORY_FILE_SIZE,\n        max_part_header_bytes: int = DEFAULT_MAX_PART_HEADER_BYTES,\n        max_part_header_lines: int = DEFAULT_MAX_PART_HEADER_LINES,\n        min_free_disk_bytes: int = DEFAULT_MIN_FREE_DISK_BYTES,\n    ) -> FormData:\n        \"\"\"\n        Parse form data from the request body.\n\n        Supports both application/x-www-form-urlencoded and multipart/form-data.\n\n        Args:\n            files: If True, store file uploads; if False (default), discard them\n            max_file_size: Maximum size per file in bytes (default 50MB)\n            max_request_size: Maximum total request size in bytes (default 100MB)\n            max_fields: Maximum number of form fields (default 1000)\n            max_files: Maximum number of file uploads (default 100)\n            max_parts: Maximum number of multipart parts (default max_fields + max_files)\n            max_field_size: Maximum size of a text field value in bytes (default 100KB)\n            max_memory_file_size: Threshold before files spill to disk (default 1MB)\n            max_part_header_bytes: Maximum bytes allowed in part headers (default 16KB)\n            max_part_header_lines: Maximum header lines per part (default 100)\n            min_free_disk_bytes: Minimum free bytes required in temp dir (default 50MB)\n\n        Returns:\n            FormData object with dict-like access to fields and files.\n            Use form[\"key\"] for first value, form.getlist(\"key\") for all values.\n\n        Raises:\n            BadRequest: If content-type is missing, unsupported, or parsing fails\n        \"\"\"\n        content_type = self.headers.get(\"content-type\", \"\")\n        if not content_type:\n            raise BadRequest(\n                \"Missing Content-Type header; expected application/x-www-form-urlencoded \"\n                \"or multipart/form-data\"\n            )\n\n        try:\n            return await parse_form_data(\n                receive=self.receive,\n                content_type=content_type,\n                files=files,\n                max_file_size=max_file_size,\n                max_request_size=max_request_size,\n                max_fields=max_fields,\n                max_files=max_files,\n                max_parts=max_parts,\n                max_field_size=max_field_size,\n                max_memory_file_size=max_memory_file_size,\n                max_part_header_bytes=max_part_header_bytes,\n                max_part_header_lines=max_part_header_lines,\n                min_free_disk_bytes=min_free_disk_bytes,\n            )\n        except MultipartParseError as e:\n            raise BadRequest(str(e))\n\n    @classmethod\n    def fake(cls, path_with_query_string, method=\"GET\", scheme=\"http\", url_vars=None):\n        \"\"\"Useful for constructing Request objects for tests\"\"\"\n        path, _, query_string = path_with_query_string.partition(\"?\")\n        scope = {\n            \"http_version\": \"1.1\",\n            \"method\": method,\n            \"path\": path,\n            \"raw_path\": path_with_query_string.encode(\"latin-1\"),\n            \"query_string\": query_string.encode(\"latin-1\"),\n            \"scheme\": scheme,\n            \"type\": \"http\",\n        }\n        if url_vars:\n            scope[\"url_route\"] = {\"kwargs\": url_vars}\n        return cls(scope, None)\n\n\nclass AsgiLifespan:\n    def __init__(self, app, on_startup=None, on_shutdown=None):\n        self.app = app\n        on_startup = on_startup or []\n        on_shutdown = on_shutdown or []\n        if not isinstance(on_startup or [], list):\n            on_startup = [on_startup]\n        if not isinstance(on_shutdown or [], list):\n            on_shutdown = [on_shutdown]\n        self.on_startup = on_startup\n        self.on_shutdown = on_shutdown\n\n    async def __call__(self, scope, receive, send):\n        if scope[\"type\"] == \"lifespan\":\n            while True:\n                message = await receive()\n                if message[\"type\"] == \"lifespan.startup\":\n                    for fn in self.on_startup:\n                        await fn()\n                    await send({\"type\": \"lifespan.startup.complete\"})\n                elif message[\"type\"] == \"lifespan.shutdown\":\n                    for fn in self.on_shutdown:\n                        await fn()\n                    await send({\"type\": \"lifespan.shutdown.complete\"})\n                    return\n        else:\n            await self.app(scope, receive, send)\n\n\nclass AsgiStream:\n    def __init__(self, stream_fn, status=200, headers=None, content_type=\"text/plain\"):\n        self.stream_fn = stream_fn\n        self.status = status\n        self.headers = headers or {}\n        self.content_type = content_type\n\n    async def asgi_send(self, send):\n        # Remove any existing content-type header\n        headers = {k: v for k, v in self.headers.items() if k.lower() != \"content-type\"}\n        headers[\"content-type\"] = self.content_type\n        await send(\n            {\n                \"type\": \"http.response.start\",\n                \"status\": self.status,\n                \"headers\": [\n                    [key.encode(\"utf-8\"), value.encode(\"utf-8\")]\n                    for key, value in headers.items()\n                ],\n            }\n        )\n        w = AsgiWriter(send)\n        await self.stream_fn(w)\n        await send({\"type\": \"http.response.body\", \"body\": b\"\"})\n\n\nclass AsgiWriter:\n    def __init__(self, send):\n        self.send = send\n\n    async def write(self, chunk):\n        await self.send(\n            {\n                \"type\": \"http.response.body\",\n                \"body\": chunk.encode(\"utf-8\"),\n                \"more_body\": True,\n            }\n        )\n\n\nasync def asgi_send_json(send, info, status=200, headers=None):\n    headers = headers or {}\n    await asgi_send(\n        send,\n        json.dumps(info),\n        status=status,\n        headers=headers,\n        content_type=\"application/json; charset=utf-8\",\n    )\n\n\nasync def asgi_send_html(send, html, status=200, headers=None):\n    headers = headers or {}\n    await asgi_send(\n        send,\n        html,\n        status=status,\n        headers=headers,\n        content_type=\"text/html; charset=utf-8\",\n    )\n\n\nasync def asgi_send_redirect(send, location, status=302):\n    # Prevent open redirect vulnerability: strip multiple leading slashes\n    # //example.com would be interpreted as a protocol-relative URL (e.g., https://example.com/)\n    location = re.sub(r\"^/+\", \"/\", location)\n    await asgi_send(\n        send,\n        \"\",\n        status=status,\n        headers={\"Location\": location},\n        content_type=\"text/html; charset=utf-8\",\n    )\n\n\nasync def asgi_send(send, content, status, headers=None, content_type=\"text/plain\"):\n    await asgi_start(send, status, headers, content_type)\n    await send({\"type\": \"http.response.body\", \"body\": content.encode(\"utf-8\")})\n\n\nasync def asgi_start(send, status, headers=None, content_type=\"text/plain\"):\n    headers = headers or {}\n    # Remove any existing content-type header\n    headers = {k: v for k, v in headers.items() if k.lower() != \"content-type\"}\n    headers[\"content-type\"] = content_type\n    await send(\n        {\n            \"type\": \"http.response.start\",\n            \"status\": status,\n            \"headers\": [\n                [key.encode(\"latin1\"), value.encode(\"latin1\")]\n                for key, value in headers.items()\n            ],\n        }\n    )\n\n\nasync def asgi_send_file(\n    send, filepath, filename=None, content_type=None, chunk_size=4096, headers=None\n):\n    headers = headers or {}\n    if filename:\n        headers[\"content-disposition\"] = f'attachment; filename=\"{filename}\"'\n\n    first = True\n    headers[\"content-length\"] = str((await aiofiles.os.stat(str(filepath))).st_size)\n    async with aiofiles.open(str(filepath), mode=\"rb\") as fp:\n        if first:\n            await asgi_start(\n                send,\n                200,\n                headers,\n                content_type or guess_type(str(filepath))[0] or \"text/plain\",\n            )\n            first = False\n        more_body = True\n        while more_body:\n            chunk = await fp.read(chunk_size)\n            more_body = len(chunk) == chunk_size\n            await send(\n                {\"type\": \"http.response.body\", \"body\": chunk, \"more_body\": more_body}\n            )\n\n\ndef asgi_static(root_path, chunk_size=4096, headers=None, content_type=None):\n    root_path = Path(root_path)\n    static_headers = {}\n\n    if headers:\n        static_headers = headers.copy()\n\n    async def inner_static(request, send):\n        path = request.scope[\"url_route\"][\"kwargs\"][\"path\"]\n        headers = static_headers.copy()\n        try:\n            full_path = (root_path / path).resolve().absolute()\n        except FileNotFoundError:\n            await asgi_send_html(send, \"404: Directory not found\", 404)\n            return\n        if full_path.is_dir():\n            await asgi_send_html(send, \"403: Directory listing is not allowed\", 403)\n            return\n        # Ensure full_path is within root_path to avoid weird \"../\" tricks\n        try:\n            full_path.relative_to(root_path.resolve())\n        except ValueError:\n            await asgi_send_html(send, \"404: Path not inside root path\", 404)\n            return\n        try:\n            # Calculate ETag for filepath\n            etag = await calculate_etag(full_path, chunk_size=chunk_size)\n            headers[\"ETag\"] = etag\n            if_none_match = request.headers.get(\"if-none-match\")\n            if if_none_match and if_none_match == etag:\n                return await asgi_send(send, \"\", 304)\n            await asgi_send_file(\n                send, full_path, chunk_size=chunk_size, headers=headers\n            )\n        except FileNotFoundError:\n            await asgi_send_html(send, \"404: File not found\", 404)\n            return\n\n    return inner_static\n\n\nclass Response:\n    def __init__(self, body=None, status=200, headers=None, content_type=\"text/plain\"):\n        self.body = body\n        self.status = status\n        self.headers = headers or {}\n        self._set_cookie_headers = []\n        self.content_type = content_type\n\n    async def asgi_send(self, send):\n        headers = {}\n        headers.update(self.headers)\n        headers[\"content-type\"] = self.content_type\n        raw_headers = [\n            [key.encode(\"utf-8\"), value.encode(\"utf-8\")]\n            for key, value in headers.items()\n        ]\n        for set_cookie in self._set_cookie_headers:\n            raw_headers.append([b\"set-cookie\", set_cookie.encode(\"utf-8\")])\n        await send(\n            {\n                \"type\": \"http.response.start\",\n                \"status\": self.status,\n                \"headers\": raw_headers,\n            }\n        )\n        body = self.body\n        if not isinstance(body, bytes):\n            body = body.encode(\"utf-8\")\n        await send({\"type\": \"http.response.body\", \"body\": body})\n\n    def set_cookie(\n        self,\n        key,\n        value=\"\",\n        max_age=None,\n        expires=None,\n        path=\"/\",\n        domain=None,\n        secure=False,\n        httponly=False,\n        samesite=\"lax\",\n    ):\n        assert samesite in SAMESITE_VALUES, \"samesite should be one of {}\".format(\n            SAMESITE_VALUES\n        )\n        cookie = SimpleCookie()\n        cookie[key] = value\n        for prop_name, prop_value in (\n            (\"max_age\", max_age),\n            (\"expires\", expires),\n            (\"path\", path),\n            (\"domain\", domain),\n            (\"samesite\", samesite),\n        ):\n            if prop_value is not None:\n                cookie[key][prop_name.replace(\"_\", \"-\")] = prop_value\n        for prop_name, prop_value in ((\"secure\", secure), (\"httponly\", httponly)):\n            if prop_value:\n                cookie[key][prop_name] = True\n        self._set_cookie_headers.append(cookie.output(header=\"\").strip())\n\n    @classmethod\n    def html(cls, body, status=200, headers=None):\n        return cls(\n            body,\n            status=status,\n            headers=headers,\n            content_type=\"text/html; charset=utf-8\",\n        )\n\n    @classmethod\n    def text(cls, body, status=200, headers=None):\n        return cls(\n            str(body),\n            status=status,\n            headers=headers,\n            content_type=\"text/plain; charset=utf-8\",\n        )\n\n    @classmethod\n    def json(cls, body, status=200, headers=None, default=None):\n        return cls(\n            json.dumps(body, default=default),\n            status=status,\n            headers=headers,\n            content_type=\"application/json; charset=utf-8\",\n        )\n\n    @classmethod\n    def redirect(cls, path, status=302, headers=None):\n        headers = headers or {}\n        headers[\"Location\"] = path\n        return cls(\"\", status=status, headers=headers)\n\n\nclass AsgiFileDownload:\n    def __init__(\n        self,\n        filepath,\n        filename=None,\n        content_type=\"application/octet-stream\",\n        headers=None,\n    ):\n        self.headers = headers or {}\n        self.filepath = filepath\n        self.filename = filename\n        self.content_type = content_type\n\n    async def asgi_send(self, send):\n        return await asgi_send_file(\n            send,\n            self.filepath,\n            filename=self.filename,\n            content_type=self.content_type,\n            headers=self.headers,\n        )\n\n\nclass AsgiRunOnFirstRequest:\n    def __init__(self, asgi, on_startup):\n        assert isinstance(on_startup, list)\n        self.asgi = asgi\n        self.on_startup = on_startup\n        self._started = False\n\n    async def __call__(self, scope, receive, send):\n        if not self._started:\n            self._started = True\n            for hook in self.on_startup:\n                await hook()\n        return await self.asgi(scope, receive, send)\n"
  },
  {
    "path": "datasette/utils/baseconv.py",
    "content": "\"\"\"\nConvert numbers from base 10 integers to base X strings and back again.\n\nSample usage:\n\n>>> base20 = BaseConverter('0123456789abcdefghij')\n>>> base20.from_decimal(1234)\n'31e'\n>>> base20.to_decimal('31e')\n1234\n\nOriginally shared here: https://www.djangosnippets.org/snippets/1431/\n\"\"\"\n\n\nclass BaseConverter(object):\n    decimal_digits = \"0123456789\"\n\n    def __init__(self, digits):\n        self.digits = digits\n\n    def encode(self, i):\n        return self.convert(i, self.decimal_digits, self.digits)\n\n    def decode(self, s):\n        return int(self.convert(s, self.digits, self.decimal_digits))\n\n    def convert(number, fromdigits, todigits):\n        # Based on http://code.activestate.com/recipes/111286/\n        if str(number)[0] == \"-\":\n            number = str(number)[1:]\n            neg = 1\n        else:\n            neg = 0\n\n        # make an integer out of the number\n        x = 0\n        for digit in str(number):\n            x = x * len(fromdigits) + fromdigits.index(digit)\n\n        # create the result in base 'len(todigits)'\n        if x == 0:\n            res = todigits[0]\n        else:\n            res = \"\"\n            while x > 0:\n                digit = x % len(todigits)\n                res = todigits[digit] + res\n                x = int(x / len(todigits))\n            if neg:\n                res = \"-\" + res\n        return res\n\n    convert = staticmethod(convert)\n\n\nbin = BaseConverter(\"01\")\nhexconv = BaseConverter(\"0123456789ABCDEF\")\nbase62 = BaseConverter(\"ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvwxyz\")\n"
  },
  {
    "path": "datasette/utils/check_callable.py",
    "content": "import inspect\nimport types\nfrom typing import NamedTuple, Any\n\n\nclass CallableStatus(NamedTuple):\n    is_callable: bool\n    is_async_callable: bool\n\n\ndef check_callable(obj: Any) -> CallableStatus:\n    if not callable(obj):\n        return CallableStatus(False, False)\n\n    if isinstance(obj, type):\n        # It's a class\n        return CallableStatus(True, False)\n\n    if isinstance(obj, types.FunctionType):\n        return CallableStatus(True, inspect.iscoroutinefunction(obj))\n\n    if hasattr(obj, \"__call__\"):\n        return CallableStatus(True, inspect.iscoroutinefunction(obj.__call__))\n\n    assert False, \"obj {} is somehow callable with no __call__ method\".format(repr(obj))\n"
  },
  {
    "path": "datasette/utils/internal_db.py",
    "content": "import textwrap\nfrom datasette.utils import table_column_details\n\n\nasync def init_internal_db(db):\n    create_tables_sql = textwrap.dedent(\"\"\"\n    CREATE TABLE IF NOT EXISTS catalog_databases (\n        database_name TEXT PRIMARY KEY,\n        path TEXT,\n        is_memory INTEGER,\n        schema_version INTEGER\n    );\n    CREATE TABLE IF NOT EXISTS catalog_tables (\n        database_name TEXT,\n        table_name TEXT,\n        rootpage INTEGER,\n        sql TEXT,\n        PRIMARY KEY (database_name, table_name),\n        FOREIGN KEY (database_name) REFERENCES catalog_databases(database_name)\n    );\n    CREATE TABLE IF NOT EXISTS catalog_views (\n        database_name TEXT,\n        view_name TEXT,\n        rootpage INTEGER,\n        sql TEXT,\n        PRIMARY KEY (database_name, view_name),\n        FOREIGN KEY (database_name) REFERENCES catalog_databases(database_name)\n    );\n    CREATE TABLE IF NOT EXISTS catalog_columns (\n        database_name TEXT,\n        table_name TEXT,\n        cid INTEGER,\n        name TEXT,\n        type TEXT,\n        \"notnull\" INTEGER,\n        default_value TEXT, -- renamed from dflt_value\n        is_pk INTEGER, -- renamed from pk\n        hidden INTEGER,\n        PRIMARY KEY (database_name, table_name, name),\n        FOREIGN KEY (database_name) REFERENCES catalog_databases(database_name),\n        FOREIGN KEY (database_name, table_name) REFERENCES catalog_tables(database_name, table_name)\n    );\n    CREATE TABLE IF NOT EXISTS catalog_indexes (\n        database_name TEXT,\n        table_name TEXT,\n        seq INTEGER,\n        name TEXT,\n        \"unique\" INTEGER,\n        origin TEXT,\n        partial INTEGER,\n        PRIMARY KEY (database_name, table_name, name),\n        FOREIGN KEY (database_name) REFERENCES catalog_databases(database_name),\n        FOREIGN KEY (database_name, table_name) REFERENCES catalog_tables(database_name, table_name)\n    );\n    CREATE TABLE IF NOT EXISTS catalog_foreign_keys (\n        database_name TEXT,\n        table_name TEXT,\n        id INTEGER,\n        seq INTEGER,\n        \"table\" TEXT,\n        \"from\" TEXT,\n        \"to\" TEXT,\n        on_update TEXT,\n        on_delete TEXT,\n        match TEXT,\n        PRIMARY KEY (database_name, table_name, id, seq),\n        FOREIGN KEY (database_name) REFERENCES catalog_databases(database_name),\n        FOREIGN KEY (database_name, table_name) REFERENCES catalog_tables(database_name, table_name)\n    );\n    \"\"\").strip()\n    await db.execute_write_script(create_tables_sql)\n    await initialize_metadata_tables(db)\n\n\nasync def initialize_metadata_tables(db):\n    await db.execute_write_script(textwrap.dedent(\"\"\"\n        CREATE TABLE IF NOT EXISTS metadata_instance (\n            key text,\n            value text,\n            unique(key)\n        );\n\n        CREATE TABLE IF NOT EXISTS metadata_databases (\n            database_name text,\n            key text,\n            value text,\n            unique(database_name, key)\n        );\n\n        CREATE TABLE IF NOT EXISTS metadata_resources (\n            database_name text,\n            resource_name text,\n            key text,\n            value text,\n            unique(database_name, resource_name, key)\n        );\n\n        CREATE TABLE IF NOT EXISTS metadata_columns (\n            database_name text,\n            resource_name text,\n            column_name text,\n            key text,\n            value text,\n            unique(database_name, resource_name, column_name, key)\n        );\n\n        CREATE TABLE IF NOT EXISTS column_types (\n            database_name TEXT NOT NULL,\n            resource_name TEXT NOT NULL,\n            column_name TEXT NOT NULL,\n            column_type TEXT NOT NULL,\n            config TEXT,\n            PRIMARY KEY (database_name, resource_name, column_name)\n        );\n            \"\"\"))\n\n\nasync def populate_schema_tables(internal_db, db):\n    database_name = db.name\n\n    def delete_everything(conn):\n        conn.execute(\n            \"DELETE FROM catalog_tables WHERE database_name = ?\", [database_name]\n        )\n        conn.execute(\n            \"DELETE FROM catalog_views WHERE database_name = ?\", [database_name]\n        )\n        conn.execute(\n            \"DELETE FROM catalog_columns WHERE database_name = ?\", [database_name]\n        )\n        conn.execute(\n            \"DELETE FROM catalog_foreign_keys WHERE database_name = ?\",\n            [database_name],\n        )\n        conn.execute(\n            \"DELETE FROM catalog_indexes WHERE database_name = ?\", [database_name]\n        )\n\n    await internal_db.execute_write_fn(delete_everything)\n\n    tables = (await db.execute(\"select * from sqlite_master WHERE type = 'table'\")).rows\n    views = (await db.execute(\"select * from sqlite_master WHERE type = 'view'\")).rows\n\n    def collect_info(conn):\n        tables_to_insert = []\n        views_to_insert = []\n        columns_to_insert = []\n        foreign_keys_to_insert = []\n        indexes_to_insert = []\n\n        for view in views:\n            view_name = view[\"name\"]\n            views_to_insert.append(\n                (database_name, view_name, view[\"rootpage\"], view[\"sql\"])\n            )\n\n        for table in tables:\n            table_name = table[\"name\"]\n            tables_to_insert.append(\n                (database_name, table_name, table[\"rootpage\"], table[\"sql\"])\n            )\n            columns = table_column_details(conn, table_name)\n            columns_to_insert.extend(\n                {\n                    **{\"database_name\": database_name, \"table_name\": table_name},\n                    **column._asdict(),\n                }\n                for column in columns\n            )\n            foreign_keys = conn.execute(\n                f\"PRAGMA foreign_key_list([{table_name}])\"\n            ).fetchall()\n            foreign_keys_to_insert.extend(\n                {\n                    **{\"database_name\": database_name, \"table_name\": table_name},\n                    **dict(foreign_key),\n                }\n                for foreign_key in foreign_keys\n            )\n            indexes = conn.execute(f\"PRAGMA index_list([{table_name}])\").fetchall()\n            indexes_to_insert.extend(\n                {\n                    **{\"database_name\": database_name, \"table_name\": table_name},\n                    **dict(index),\n                }\n                for index in indexes\n            )\n        return (\n            tables_to_insert,\n            views_to_insert,\n            columns_to_insert,\n            foreign_keys_to_insert,\n            indexes_to_insert,\n        )\n\n    (\n        tables_to_insert,\n        views_to_insert,\n        columns_to_insert,\n        foreign_keys_to_insert,\n        indexes_to_insert,\n    ) = await db.execute_fn(collect_info)\n\n    await internal_db.execute_write_many(\n        \"\"\"\n        INSERT INTO catalog_tables (database_name, table_name, rootpage, sql)\n        values (?, ?, ?, ?)\n    \"\"\",\n        tables_to_insert,\n    )\n    await internal_db.execute_write_many(\n        \"\"\"\n        INSERT INTO catalog_views (database_name, view_name, rootpage, sql)\n        values (?, ?, ?, ?)\n    \"\"\",\n        views_to_insert,\n    )\n    await internal_db.execute_write_many(\n        \"\"\"\n        INSERT INTO catalog_columns (\n            database_name, table_name, cid, name, type, \"notnull\", default_value, is_pk, hidden\n        ) VALUES (\n            :database_name, :table_name, :cid, :name, :type, :notnull, :default_value, :is_pk, :hidden\n        )\n    \"\"\",\n        columns_to_insert,\n    )\n    await internal_db.execute_write_many(\n        \"\"\"\n        INSERT INTO catalog_foreign_keys (\n            database_name, table_name, \"id\", seq, \"table\", \"from\", \"to\", on_update, on_delete, match\n        ) VALUES (\n            :database_name, :table_name, :id, :seq, :table, :from, :to, :on_update, :on_delete, :match\n        )\n    \"\"\",\n        foreign_keys_to_insert,\n    )\n    await internal_db.execute_write_many(\n        \"\"\"\n        INSERT INTO catalog_indexes (\n            database_name, table_name, seq, name, \"unique\", origin, partial\n        ) VALUES (\n            :database_name, :table_name, :seq, :name, :unique, :origin, :partial\n        )\n    \"\"\",\n        indexes_to_insert,\n    )\n"
  },
  {
    "path": "datasette/utils/multipart.py",
    "content": "\"\"\"\nStreaming multipart/form-data parser for ASGI applications.\n\nSupports:\n- Streaming parsing without buffering entire body in memory\n- Files spill to disk above configurable threshold\n- Security limits on request size, file size, field count\n- Both multipart/form-data and application/x-www-form-urlencoded\n\"\"\"\n\nimport asyncio\nimport shutil\nimport tempfile\nfrom dataclasses import dataclass, field\nfrom typing import (\n    Any,\n    Callable,\n    Dict,\n    List,\n    Optional,\n    Tuple,\n    Union,\n)\nfrom urllib.parse import parse_qsl\n\n# Centralized defaults for multipart/form-data parsing\nDEFAULT_MAX_FILE_SIZE = 50 * 1024 * 1024  # 50MB\nDEFAULT_MAX_REQUEST_SIZE = 100 * 1024 * 1024  # 100MB\nDEFAULT_MAX_FIELDS = 1000\nDEFAULT_MAX_FILES = 100\n# If max_parts is not specified, it defaults to max_fields + max_files\nDEFAULT_MAX_PARTS: Optional[int] = None\nDEFAULT_MAX_FIELD_SIZE = 100 * 1024  # 100KB\nDEFAULT_MAX_MEMORY_FILE_SIZE = 1024 * 1024  # 1MB\nDEFAULT_MAX_PART_HEADER_BYTES = 16 * 1024  # 16KB\nDEFAULT_MAX_PART_HEADER_LINES = 100\nDEFAULT_MIN_FREE_DISK_BYTES = 50 * 1024 * 1024  # 50MB\n\n\nclass MultipartParseError(Exception):\n    \"\"\"Raised when multipart parsing fails.\"\"\"\n\n    pass\n\n\n@dataclass\nclass UploadedFile:\n    \"\"\"\n    Represents an uploaded file from a multipart form.\n\n    Attributes:\n        name: The form field name\n        filename: The original filename from the upload\n        content_type: The MIME type of the file\n        size: Size in bytes\n    \"\"\"\n\n    name: str\n    filename: str\n    content_type: Optional[str]\n    size: int\n    _file: tempfile.SpooledTemporaryFile = field(repr=False)\n\n    async def read(self, size: int = -1) -> bytes:\n        \"\"\"Read file contents.\"\"\"\n        return await asyncio.to_thread(self._file.read, size)\n\n    async def seek(self, offset: int, whence: int = 0) -> int:\n        \"\"\"Seek to position in file.\"\"\"\n        return await asyncio.to_thread(self._file.seek, offset, whence)\n\n    async def close(self) -> None:\n        \"\"\"Close the underlying file.\"\"\"\n        await asyncio.to_thread(self._file.close)\n\n    def close_sync(self) -> None:\n        \"\"\"Close the underlying file synchronously.\"\"\"\n        self._file.close()\n\n    async def __aenter__(self):\n        return self\n\n    async def __aexit__(self, exc_type, exc, tb):\n        await self.close()\n\n    def __del__(self):\n        try:\n            self._file.close()\n        except Exception:\n            pass\n\n\nclass FormData:\n    \"\"\"\n    Container for parsed form data, supporting both fields and files.\n\n    Provides dict-like access with support for multiple values per key.\n    \"\"\"\n\n    def __init__(self):\n        self._data: List[Tuple[str, Union[str, UploadedFile]]] = []\n\n    def append(self, key: str, value: Union[str, UploadedFile]) -> None:\n        \"\"\"Add a key-value pair.\"\"\"\n        self._data.append((key, value))\n\n    def __getitem__(self, key: str) -> Union[str, UploadedFile]:\n        \"\"\"Get the first value for a key.\"\"\"\n        for k, v in self._data:\n            if k == key:\n                return v\n        raise KeyError(key)\n\n    def get(self, key: str, default: Any = None) -> Optional[Union[str, UploadedFile]]:\n        \"\"\"Get the first value for a key, or default if not found.\"\"\"\n        try:\n            return self[key]\n        except KeyError:\n            return default\n\n    def getlist(self, key: str) -> List[Union[str, UploadedFile]]:\n        \"\"\"Get all values for a key.\"\"\"\n        return [v for k, v in self._data if k == key]\n\n    def __contains__(self, key: str) -> bool:\n        \"\"\"Check if key exists.\"\"\"\n        return any(k == key for k, _ in self._data)\n\n    def __len__(self) -> int:\n        \"\"\"Return number of items.\"\"\"\n        return len(self._data)\n\n    def __iter__(self):\n        \"\"\"Iterate over unique keys.\"\"\"\n        seen = set()\n        for k, _ in self._data:\n            if k not in seen:\n                seen.add(k)\n                yield k\n\n    def keys(self):\n        \"\"\"Return unique keys.\"\"\"\n        return list(self)\n\n    def items(self) -> List[Tuple[str, Union[str, UploadedFile]]]:\n        \"\"\"Return all key-value pairs.\"\"\"\n        return list(self._data)\n\n    def values(self) -> List[Union[str, UploadedFile]]:\n        \"\"\"Return all values.\"\"\"\n        return [v for _, v in self._data]\n\n    def _uploaded_files(self) -> List[UploadedFile]:\n        \"\"\"Return UploadedFile instances contained in this form.\"\"\"\n        return [v for _, v in self._data if isinstance(v, UploadedFile)]\n\n    def close(self) -> None:\n        \"\"\"\n        Close any uploaded files.\n\n        This provides deterministic cleanup for spooled temp files.\n        \"\"\"\n        for uploaded in self._uploaded_files():\n            try:\n                uploaded.close_sync()\n            except Exception:\n                # Best-effort cleanup; ignore close errors\n                pass\n\n    async def aclose(self) -> None:\n        \"\"\"Asynchronously close any uploaded files.\"\"\"\n        for uploaded in self._uploaded_files():\n            try:\n                await uploaded.close()\n            except Exception:\n                # Best-effort cleanup; ignore close errors\n                pass\n\n    def __enter__(self):\n        return self\n\n    def __exit__(self, exc_type, exc, tb):\n        self.close()\n\n    async def __aenter__(self):\n        return self\n\n    async def __aexit__(self, exc_type, exc, tb):\n        await self.aclose()\n\n\ndef parse_content_disposition(header: str) -> Dict[str, Optional[str]]:\n    \"\"\"\n    Parse Content-Disposition header value.\n\n    Returns dict with 'name', 'filename' keys (filename may be None).\n    \"\"\"\n    result: Dict[str, Optional[str]] = {\"name\": None, \"filename\": None}\n\n    # Split on semicolons, handling quoted strings\n    parts = []\n    current = \"\"\n    in_quotes = False\n    i = 0\n    while i < len(header):\n        char = header[i]\n        if char == '\"' and (i == 0 or header[i - 1] != \"\\\\\"):\n            in_quotes = not in_quotes\n            current += char\n        elif char == \";\" and not in_quotes:\n            parts.append(current.strip())\n            current = \"\"\n        else:\n            current += char\n        i += 1\n    if current.strip():\n        parts.append(current.strip())\n\n    for part in parts[1:]:  # Skip the \"form-data\" part\n        if \"=\" not in part:\n            continue\n\n        key, _, value = part.partition(\"=\")\n        key = key.strip().lower()\n        value = value.strip()\n\n        # Handle filename* (RFC 5987 encoding)\n        if key == \"filename*\":\n            # Format: utf-8''encoded_filename or charset'language'encoded_filename\n            if \"'\" in value:\n                parts_star = value.split(\"'\", 2)\n                if len(parts_star) >= 3:\n                    # charset = parts_star[0]\n                    # language = parts_star[1]\n                    encoded = parts_star[2]\n                    # URL decode\n                    try:\n                        from urllib.parse import unquote\n\n                        result[\"filename\"] = unquote(encoded, encoding=\"utf-8\")\n                    except Exception:\n                        pass\n            continue\n\n        # Remove quotes if present\n        if value.startswith('\"') and value.endswith('\"'):\n            value = value[1:-1]\n            # Unescape backslash sequences\n            value = value.replace('\\\\\"', '\"').replace(\"\\\\\\\\\", \"\\\\\")\n\n        if key == \"name\":\n            result[\"name\"] = value\n        elif key == \"filename\":\n            # Only set if filename* hasn't already set it\n            if result[\"filename\"] is None:\n                # Strip path components (security)\n                # Handle both Unix and Windows paths\n                value = value.replace(\"\\\\\", \"/\")\n                if \"/\" in value:\n                    value = value.rsplit(\"/\", 1)[-1]\n                result[\"filename\"] = value\n\n    return result\n\n\ndef parse_content_type(header: str) -> Tuple[str, Dict[str, str]]:\n    \"\"\"\n    Parse Content-Type header value.\n\n    Returns (media_type, parameters_dict).\n    \"\"\"\n    parts = header.split(\";\")\n    media_type = parts[0].strip().lower()\n    params = {}\n\n    for part in parts[1:]:\n        part = part.strip()\n        if \"=\" in part:\n            key, _, value = part.partition(\"=\")\n            key = key.strip().lower()\n            value = value.strip()\n            # Remove quotes if present\n            if value.startswith('\"') and value.endswith('\"'):\n                value = value[1:-1]\n            params[key] = value\n\n    return media_type, params\n\n\nclass MultipartParser:\n    \"\"\"\n    Streaming multipart/form-data parser.\n\n    Processes the body chunk by chunk without loading everything into memory.\n    \"\"\"\n\n    # Parser states\n    STATE_PREAMBLE = 0\n    STATE_HEADER = 1\n    STATE_BODY = 2\n    STATE_DONE = 3\n\n    def __init__(\n        self,\n        boundary: bytes,\n        max_file_size: int = DEFAULT_MAX_FILE_SIZE,\n        max_request_size: int = DEFAULT_MAX_REQUEST_SIZE,\n        max_fields: int = DEFAULT_MAX_FIELDS,\n        max_files: int = DEFAULT_MAX_FILES,\n        max_parts: Optional[int] = DEFAULT_MAX_PARTS,\n        max_field_size: int = DEFAULT_MAX_FIELD_SIZE,\n        max_memory_file_size: int = DEFAULT_MAX_MEMORY_FILE_SIZE,\n        max_part_header_bytes: int = DEFAULT_MAX_PART_HEADER_BYTES,\n        max_part_header_lines: int = DEFAULT_MAX_PART_HEADER_LINES,\n        min_free_disk_bytes: int = DEFAULT_MIN_FREE_DISK_BYTES,\n        handle_files: bool = False,\n    ):\n        self.boundary = b\"--\" + boundary\n        self.end_boundary = self.boundary + b\"--\"\n        self.max_file_size = max_file_size\n        self.max_request_size = max_request_size\n        self.max_fields = max_fields\n        self.max_files = max_files\n        # If not specified, tie max_parts to the other cardinality limits\n        if max_parts is None:\n            max_parts = max_fields + max_files\n        self.max_parts = max_parts\n        self.max_field_size = max_field_size\n        self.max_memory_file_size = max_memory_file_size\n        self.max_part_header_bytes = max_part_header_bytes\n        self.max_part_header_lines = max_part_header_lines\n        self.min_free_disk_bytes = min_free_disk_bytes\n        self.handle_files = handle_files\n\n        self.state = self.STATE_PREAMBLE\n        self.buffer = bytearray()\n        self.total_bytes = 0\n        self.field_count = 0\n        self.file_count = 0\n        self.part_count = 0\n        self.current_part_size = 0\n        self.current_header_bytes = 0\n        self.current_header_lines = 0\n\n        self.form_data = FormData()\n        self._disk_check_interval_bytes = 1024 * 1024  # 1MB between disk checks\n        self._bytes_since_disk_check = 0\n        self._tempdir = tempfile.gettempdir()\n\n        # Current part state\n        self.current_headers: Dict[str, str] = {}\n        self.current_file: Optional[tempfile.SpooledTemporaryFile] = None\n        self.current_body = bytearray()\n        self.current_name: Optional[str] = None\n        self.current_filename: Optional[str] = None\n        self.current_content_type: Optional[str] = None\n\n    def feed(self, chunk: bytes) -> None:\n        \"\"\"Feed a chunk of data to the parser.\"\"\"\n        self.total_bytes += len(chunk)\n        if self.total_bytes > self.max_request_size:\n            raise MultipartParseError(\"Request body too large\")\n\n        self.buffer.extend(chunk)\n        self._process()\n\n    def _process(self) -> None:\n        \"\"\"Process buffered data.\"\"\"\n        while True:\n            if self.state == self.STATE_PREAMBLE:\n                if not self._process_preamble():\n                    break\n            elif self.state == self.STATE_HEADER:\n                if not self._process_header():\n                    break\n            elif self.state == self.STATE_BODY:\n                if not self._process_body():\n                    break\n            elif self.state == self.STATE_DONE:\n                break\n\n    def _process_preamble(self) -> bool:\n        \"\"\"Skip preamble and find first boundary.\"\"\"\n        # Look for boundary (could be at start or after preamble)\n        # Try both \\r\\n prefixed and bare boundary at start\n        idx = self.buffer.find(self.boundary)\n        if idx == -1:\n            # Keep potential partial boundary at end\n            keep = len(self.boundary) - 1\n            if len(self.buffer) > keep:\n                self.buffer = self.buffer[-keep:]\n            return False\n\n        # Found boundary, skip to after it\n        after_boundary = idx + len(self.boundary)\n\n        # Check for end boundary\n        if self.buffer[idx : idx + len(self.end_boundary)] == self.end_boundary:\n            self.state = self.STATE_DONE\n            return False\n\n        # Skip CRLF or LF after boundary\n        if after_boundary < len(self.buffer):\n            if self.buffer[after_boundary : after_boundary + 2] == b\"\\r\\n\":\n                after_boundary += 2\n            elif self.buffer[after_boundary : after_boundary + 1] == b\"\\n\":\n                after_boundary += 1\n\n        self.buffer = self.buffer[after_boundary:]\n        self.state = self.STATE_HEADER\n        self.current_headers = {}\n        self.current_header_bytes = 0\n        self.current_header_lines = 0\n        return True\n\n    def _process_header(self) -> bool:\n        \"\"\"Parse part headers.\"\"\"\n        while True:\n            # Look for end of header line\n            crlf_idx = self.buffer.find(b\"\\r\\n\")\n            lf_idx = self.buffer.find(b\"\\n\")\n\n            if crlf_idx == -1 and lf_idx == -1:\n                # Guard against unbounded header buffering if no newline is ever sent\n                if len(self.buffer) > self.max_part_header_bytes:\n                    raise MultipartParseError(\"Part headers too large\")\n                return False  # Need more data\n\n            # Use whichever comes first\n            if crlf_idx != -1 and (lf_idx == -1 or crlf_idx < lf_idx):\n                idx = crlf_idx\n                line_end_len = 2\n            else:\n                idx = lf_idx\n                line_end_len = 1\n\n            line = self.buffer[:idx]\n            self.buffer = self.buffer[idx + line_end_len :]\n\n            self.current_header_lines += 1\n            self.current_header_bytes += idx + line_end_len\n            if (\n                self.current_header_lines > self.max_part_header_lines\n                or self.current_header_bytes > self.max_part_header_bytes\n            ):\n                raise MultipartParseError(\"Part headers too large\")\n\n            if not line:\n                # Empty line = end of headers\n                self._start_body()\n                self.state = self.STATE_BODY\n                return True\n\n            # Parse header\n            try:\n                line_str = line.decode(\"utf-8\", errors=\"replace\")\n            except Exception:\n                line_str = line.decode(\"latin-1\")\n\n            if \":\" in line_str:\n                name, _, value = line_str.partition(\":\")\n                self.current_headers[name.strip().lower()] = value.strip()\n\n    def _start_body(self) -> None:\n        \"\"\"Initialize body parsing for current part.\"\"\"\n        self.part_count += 1\n        if self.part_count > self.max_parts:\n            raise MultipartParseError(\"Too many parts\")\n\n        # Parse Content-Disposition\n        cd = self.current_headers.get(\"content-disposition\", \"\")\n        parsed = parse_content_disposition(cd)\n        self.current_name = parsed.get(\"name\")\n        self.current_filename = parsed.get(\"filename\")\n        self.current_content_type = self.current_headers.get(\"content-type\")\n        self.current_part_size = 0\n\n        if self.current_filename is not None:\n            # It's a file\n            self.file_count += 1\n            if self.file_count > self.max_files:\n                raise MultipartParseError(\"Too many files\")\n            if self.handle_files:\n                self.current_file = tempfile.SpooledTemporaryFile(\n                    max_size=self.max_memory_file_size\n                )\n            else:\n                # Will discard file content\n                self.current_file = None\n        else:\n            # It's a text field\n            self.field_count += 1\n            if self.field_count > self.max_fields:\n                raise MultipartParseError(\"Too many fields\")\n            self.current_body = bytearray()\n            self.current_file = None\n\n        # Check disk space before allocating a spooled temp file\n        if self.current_filename is not None and self.handle_files:\n            self._ensure_disk_space()\n\n    def _process_body(self) -> bool:\n        \"\"\"Process body data for current part.\"\"\"\n        # Look for boundary in buffer\n        # Need to handle boundary potentially split across chunks\n\n        # The boundary is preceded by \\r\\n (or \\n for lenient parsing)\n        search_boundary = b\"\\r\\n\" + self.boundary\n\n        idx = self.buffer.find(search_boundary)\n        if idx == -1:\n            # Try LF-only boundary (lenient)\n            search_boundary_lf = b\"\\n\" + self.boundary\n            idx = self.buffer.find(search_boundary_lf)\n            if idx != -1:\n                search_boundary = search_boundary_lf\n\n        if idx == -1:\n            # No boundary found yet\n            # Keep potential partial boundary at end of buffer\n            safe_len = len(self.buffer) - len(search_boundary) - 1\n            if safe_len > 0:\n                safe_data = self.buffer[:safe_len]\n                self._write_body_data(bytes(safe_data))\n                self.buffer = self.buffer[safe_len:]\n            return False\n\n        # Found boundary - write remaining body data\n        body_data = self.buffer[:idx]\n        self._write_body_data(bytes(body_data))\n\n        # Move past the boundary\n        after_boundary = idx + len(search_boundary)\n\n        # Check for end boundary\n        remaining = self.buffer[after_boundary:]\n        if remaining.startswith(b\"--\"):\n            # End boundary\n            self._finish_part()\n            self.state = self.STATE_DONE\n            return False\n\n        # Skip CRLF or LF after boundary\n        if remaining.startswith(b\"\\r\\n\"):\n            after_boundary += 2\n        elif remaining.startswith(b\"\\n\"):\n            after_boundary += 1\n\n        self.buffer = self.buffer[after_boundary:]\n        self._finish_part()\n        self.state = self.STATE_HEADER\n        self.current_headers = {}\n        self.current_header_bytes = 0\n        self.current_header_lines = 0\n        return True\n\n    def _write_body_data(self, data: bytes) -> None:\n        \"\"\"Write data to current part body.\"\"\"\n        if not data:\n            return\n\n        self.current_part_size += len(data)\n\n        if self.current_filename is not None:\n            # File data\n            if self.current_part_size > self.max_file_size:\n                raise MultipartParseError(\"File too large\")\n            if self.handle_files and self.current_file:\n                self._bytes_since_disk_check += len(data)\n                if self._bytes_since_disk_check >= self._disk_check_interval_bytes:\n                    self._ensure_disk_space()\n                    self._bytes_since_disk_check = 0\n                self.current_file.write(data)\n            # else: discard file data\n        else:\n            # Field data\n            if self.current_part_size > self.max_field_size:\n                raise MultipartParseError(\"Field value too large\")\n            self.current_body.extend(data)\n\n    def _finish_part(self) -> None:\n        \"\"\"Finalize current part and add to form data.\"\"\"\n        if self.current_name is None:\n            return\n\n        if self.current_filename is not None:\n            # File\n            if self.handle_files and self.current_file:\n                self.current_file.seek(0)\n                uploaded = UploadedFile(\n                    name=self.current_name,\n                    filename=self.current_filename,\n                    content_type=self.current_content_type,\n                    size=self.current_part_size,\n                    _file=self.current_file,\n                )\n                self.form_data.append(self.current_name, uploaded)\n            # else: file was discarded\n        else:\n            # Text field\n            try:\n                value = bytes(self.current_body).decode(\"utf-8\")\n            except UnicodeDecodeError:\n                value = bytes(self.current_body).decode(\"latin-1\")\n            self.form_data.append(self.current_name, value)\n\n        # Reset part state\n        self.current_file = None\n        self.current_body = bytearray()\n        self.current_name = None\n        self.current_filename = None\n        self.current_content_type = None\n\n    def finalize(self) -> FormData:\n        \"\"\"Finalize parsing and return form data.\"\"\"\n        # Process any remaining data\n        self._process()\n        if self.state != self.STATE_DONE:\n            raise MultipartParseError(\n                \"Truncated multipart body (missing closing boundary)\"\n            )\n        return self.form_data\n\n    def _ensure_disk_space(self) -> None:\n        \"\"\"\n        Ensure there is enough free space on the temp filesystem.\n\n        This is a best-effort guard against filling the disk with uploads.\n        \"\"\"\n        if not self.handle_files:\n            return\n        if self.min_free_disk_bytes <= 0:\n            return\n        free_bytes = shutil.disk_usage(self._tempdir).free\n        if free_bytes < self.min_free_disk_bytes:\n            raise MultipartParseError(\"Insufficient disk space for uploads\")\n\n\nasync def parse_form_data(\n    receive: Callable,\n    content_type: str,\n    files: bool = False,\n    max_file_size: int = DEFAULT_MAX_FILE_SIZE,\n    max_request_size: int = DEFAULT_MAX_REQUEST_SIZE,\n    max_fields: int = DEFAULT_MAX_FIELDS,\n    max_files: int = DEFAULT_MAX_FILES,\n    max_parts: Optional[int] = DEFAULT_MAX_PARTS,\n    max_field_size: int = DEFAULT_MAX_FIELD_SIZE,\n    max_memory_file_size: int = DEFAULT_MAX_MEMORY_FILE_SIZE,\n    max_part_header_bytes: int = DEFAULT_MAX_PART_HEADER_BYTES,\n    max_part_header_lines: int = DEFAULT_MAX_PART_HEADER_LINES,\n    min_free_disk_bytes: int = DEFAULT_MIN_FREE_DISK_BYTES,\n) -> FormData:\n    \"\"\"\n    Parse form data from an ASGI receive callable.\n\n    Supports both application/x-www-form-urlencoded and multipart/form-data.\n\n    Args:\n        receive: ASGI receive callable\n        content_type: Content-Type header value\n        files: If True, store file uploads; if False, discard them\n        max_file_size: Maximum size per file in bytes\n        max_request_size: Maximum total request size in bytes\n        max_fields: Maximum number of form fields\n        max_files: Maximum number of file uploads\n        max_field_size: Maximum size of a text field value\n        max_memory_file_size: File size threshold before spilling to disk\n\n    Returns:\n        FormData object containing parsed fields and files\n    \"\"\"\n    media_type, params = parse_content_type(content_type)\n\n    if media_type == \"application/x-www-form-urlencoded\":\n        # Read entire body for URL-encoded forms (they're typically small)\n        body = bytearray()\n        total = 0\n        while True:\n            message = await receive()\n            message_type = message.get(\"type\")\n            if message_type == \"http.disconnect\":\n                raise MultipartParseError(\"Client disconnected during request body\")\n            if message_type is not None and message_type != \"http.request\":\n                continue\n            chunk = message.get(\"body\", b\"\")\n            total += len(chunk)\n            if total > max_request_size:\n                raise MultipartParseError(\"Request body too large\")\n            body.extend(chunk)\n            if not message.get(\"more_body\", False):\n                break\n\n        form_data = FormData()\n        try:\n            pairs = parse_qsl(bytes(body).decode(\"utf-8\"), keep_blank_values=True)\n        except UnicodeDecodeError:\n            pairs = parse_qsl(bytes(body).decode(\"latin-1\"), keep_blank_values=True)\n\n        for key, value in pairs:\n            form_data.append(key, value)\n\n        return form_data\n\n    elif media_type == \"multipart/form-data\":\n        boundary = params.get(\"boundary\")\n        if not boundary:\n            raise MultipartParseError(\"Missing boundary in Content-Type\")\n\n        parser = MultipartParser(\n            boundary=boundary.encode(\"utf-8\"),\n            max_file_size=max_file_size,\n            max_request_size=max_request_size,\n            max_fields=max_fields,\n            max_files=max_files,\n            max_parts=max_parts,\n            max_field_size=max_field_size,\n            max_memory_file_size=max_memory_file_size,\n            max_part_header_bytes=max_part_header_bytes,\n            max_part_header_lines=max_part_header_lines,\n            min_free_disk_bytes=min_free_disk_bytes,\n            handle_files=files,\n        )\n\n        # Stream body through parser\n        batch_target = 64 * 1024\n        batch = bytearray()\n\n        async def flush_batch() -> None:\n            if batch:\n                data = bytes(batch)\n                batch.clear()\n                await asyncio.to_thread(parser.feed, data)\n\n        while True:\n            message = await receive()\n            message_type = message.get(\"type\")\n            if message_type == \"http.disconnect\":\n                raise MultipartParseError(\"Client disconnected during request body\")\n            if message_type is not None and message_type != \"http.request\":\n                continue\n            chunk = message.get(\"body\", b\"\")\n            if chunk:\n                batch.extend(chunk)\n                if len(batch) >= batch_target:\n                    await flush_batch()\n            if not message.get(\"more_body\", False):\n                break\n\n        await flush_batch()\n        return await asyncio.to_thread(parser.finalize)\n\n    else:\n        raise MultipartParseError(\n            f\"Unsupported Content-Type: {media_type}. \"\n            \"Expected application/x-www-form-urlencoded or multipart/form-data\"\n        )\n"
  },
  {
    "path": "datasette/utils/permissions.py",
    "content": "# perm_utils.py\nfrom __future__ import annotations\n\nimport json\nfrom typing import Any, Dict, Iterable, List, Sequence, Tuple\nimport sqlite3\n\nfrom datasette.permissions import PermissionSQL\nfrom datasette.plugins import pm\nfrom datasette.utils import await_me_maybe\n\n# Sentinel object to indicate permission checks should be skipped\nSKIP_PERMISSION_CHECKS = object()\n\n\nasync def gather_permission_sql_from_hooks(\n    *, datasette, actor: dict | None, action: str\n) -> List[PermissionSQL] | object:\n    \"\"\"Collect PermissionSQL objects from the permission_resources_sql hook.\n\n    Ensures that each returned PermissionSQL has a populated ``source``.\n\n    Returns SKIP_PERMISSION_CHECKS sentinel if skip_permission_checks context variable\n    is set, signaling that all permission checks should be bypassed.\n    \"\"\"\n    from datasette.permissions import _skip_permission_checks\n\n    # Check if we should skip permission checks BEFORE calling hooks\n    # This avoids creating unawaited coroutines\n    if _skip_permission_checks.get():\n        return SKIP_PERMISSION_CHECKS\n\n    hook_caller = pm.hook.permission_resources_sql\n    hookimpls = hook_caller.get_hookimpls()\n    hook_results = list(hook_caller(datasette=datasette, actor=actor, action=action))\n\n    collected: List[PermissionSQL] = []\n    actor_json = json.dumps(actor) if actor is not None else None\n    actor_id = actor.get(\"id\") if isinstance(actor, dict) else None\n\n    for index, result in enumerate(hook_results):\n        hookimpl = hookimpls[index]\n        resolved = await await_me_maybe(result)\n        default_source = _plugin_name_from_hookimpl(hookimpl)\n        for permission_sql in _iter_permission_sql_from_result(resolved, action=action):\n            if not permission_sql.source:\n                permission_sql.source = default_source\n            params = permission_sql.params or {}\n            params.setdefault(\"action\", action)\n            params.setdefault(\"actor\", actor_json)\n            params.setdefault(\"actor_id\", actor_id)\n            collected.append(permission_sql)\n\n    return collected\n\n\ndef _plugin_name_from_hookimpl(hookimpl) -> str:\n    if getattr(hookimpl, \"plugin_name\", None):\n        return hookimpl.plugin_name\n    plugin = getattr(hookimpl, \"plugin\", None)\n    if hasattr(plugin, \"__name__\"):\n        return plugin.__name__\n    return repr(plugin)\n\n\ndef _iter_permission_sql_from_result(\n    result: Any, *, action: str\n) -> Iterable[PermissionSQL]:\n    if result is None:\n        return []\n    if isinstance(result, PermissionSQL):\n        return [result]\n    if isinstance(result, (list, tuple)):\n        collected: List[PermissionSQL] = []\n        for item in result:\n            collected.extend(_iter_permission_sql_from_result(item, action=action))\n        return collected\n    if callable(result):\n        permission_sql = result(action)  # type: ignore[call-arg]\n        return _iter_permission_sql_from_result(permission_sql, action=action)\n    raise TypeError(\n        \"Plugin providers must return PermissionSQL instances, sequences, or callables\"\n    )\n\n\n# -----------------------------\n# Plugin interface & utilities\n# -----------------------------\n\n\ndef build_rules_union(\n    actor: dict | None, plugins: Sequence[PermissionSQL]\n) -> Tuple[str, Dict[str, Any]]:\n    \"\"\"\n    Compose plugin SQL into a UNION ALL.\n\n    Returns:\n      union_sql: a SELECT with columns (parent, child, allow, reason, source_plugin)\n      params:    dict of bound parameters including :actor (JSON), :actor_id, and plugin params\n\n    Note: Plugins are responsible for ensuring their parameter names don't conflict.\n    The system reserves these parameter names: :actor, :actor_id, :action, :filter_parent\n    Plugin parameters should be prefixed with a unique identifier (e.g., source name).\n    \"\"\"\n    parts: List[str] = []\n    actor_json = json.dumps(actor) if actor else None\n    actor_id = actor.get(\"id\") if actor else None\n    params: Dict[str, Any] = {\"actor\": actor_json, \"actor_id\": actor_id}\n\n    for p in plugins:\n        # No namespacing - just use plugin params as-is\n        params.update(p.params or {})\n\n        # Skip plugins that only provide restriction_sql (no permission rules)\n        if p.sql is None:\n            continue\n\n        parts.append(f\"\"\"\n            SELECT parent, child, allow, reason, '{p.source}' AS source_plugin FROM (\n                {p.sql}\n            )\n            \"\"\".strip())\n\n    if not parts:\n        # Empty UNION that returns no rows\n        union_sql = \"SELECT NULL parent, NULL child, NULL allow, NULL reason, 'none' source_plugin WHERE 0\"\n    else:\n        union_sql = \"\\nUNION ALL\\n\".join(parts)\n\n    return union_sql, params\n\n\n# -----------------------------------------------\n# Core resolvers (no temp tables, no custom UDFs)\n# -----------------------------------------------\n\n\nasync def resolve_permissions_from_catalog(\n    db,\n    actor: dict | None,\n    plugins: Sequence[Any],\n    action: str,\n    candidate_sql: str,\n    candidate_params: Dict[str, Any] | None = None,\n    *,\n    implicit_deny: bool = True,\n) -> List[Dict[str, Any]]:\n    \"\"\"\n    Resolve permissions by embedding the provided *candidate_sql* in a CTE.\n\n    Expectations:\n      - candidate_sql SELECTs: parent TEXT, child TEXT\n        (Use child=NULL for parent-scoped actions like \"execute-sql\".)\n      - *db* exposes: rows = await db.execute(sql, params)\n        where rows is an iterable of sqlite3.Row\n      - plugins: hook results handled by await_me_maybe - can be sync/async,\n        single PermissionSQL, list, or callable returning PermissionSQL\n      - actor is the actor dict (or None), made available as :actor (JSON), :actor_id, and :action\n\n    Decision policy:\n      1) Specificity first: child (depth=2) > parent (depth=1) > root (depth=0)\n      2) Within the same depth: deny (0) beats allow (1)\n      3) If no matching rule:\n         - implicit_deny=True  -> treat as allow=0, reason='implicit deny'\n         - implicit_deny=False -> allow=None, reason=None\n\n    Returns: list of dict rows\n      - parent, child, allow, reason, source_plugin, depth\n      - resource (rendered \"/parent/child\" or \"/parent\" or \"/\")\n    \"\"\"\n    resolved_plugins: List[PermissionSQL] = []\n    restriction_sqls: List[str] = []\n\n    for plugin in plugins:\n        if callable(plugin) and not isinstance(plugin, PermissionSQL):\n            resolved = plugin(action)  # type: ignore[arg-type]\n        else:\n            resolved = plugin  # type: ignore[assignment]\n        if not isinstance(resolved, PermissionSQL):\n            raise TypeError(\"Plugin providers must return PermissionSQL instances\")\n        resolved_plugins.append(resolved)\n\n        # Collect restriction SQL filters\n        if resolved.restriction_sql:\n            restriction_sqls.append(resolved.restriction_sql)\n\n    union_sql, rule_params = build_rules_union(actor, resolved_plugins)\n    all_params = {\n        **(candidate_params or {}),\n        **rule_params,\n        \"action\": action,\n    }\n\n    sql = f\"\"\"\n    WITH\n    cands AS (\n        {candidate_sql}\n    ),\n    rules AS (\n        {union_sql}\n    ),\n    matched AS (\n        SELECT\n            c.parent, c.child,\n            r.allow, r.reason, r.source_plugin,\n            CASE\n              WHEN r.child  IS NOT NULL THEN 2  -- child-level (most specific)\n              WHEN r.parent IS NOT NULL THEN 1  -- parent-level\n              ELSE 0                            -- root/global\n            END AS depth\n        FROM cands c\n        JOIN rules r\n          ON (r.parent IS NULL OR r.parent = c.parent)\n         AND (r.child  IS NULL OR r.child  = c.child)\n    ),\n    ranked AS (\n        SELECT *,\n               ROW_NUMBER() OVER (\n                 PARTITION BY parent, child\n                 ORDER BY\n                   depth DESC,                          -- specificity first\n                   CASE WHEN allow=0 THEN 0 ELSE 1 END, -- then deny over allow at same depth\n                   source_plugin                        -- stable tie-break\n               ) AS rn\n        FROM matched\n    ),\n    winner AS (\n        SELECT parent, child,\n               allow, reason, source_plugin, depth\n        FROM ranked WHERE rn = 1\n    )\n    SELECT\n      c.parent, c.child,\n      COALESCE(w.allow, CASE WHEN :implicit_deny THEN 0 ELSE NULL END) AS allow,\n      COALESCE(w.reason, CASE WHEN :implicit_deny THEN 'implicit deny' ELSE NULL END) AS reason,\n      w.source_plugin,\n      COALESCE(w.depth, -1) AS depth,\n      :action AS action,\n      CASE\n        WHEN c.parent IS NULL THEN '/'\n        WHEN c.child  IS NULL THEN '/' || c.parent\n        ELSE '/' || c.parent || '/' || c.child\n      END AS resource\n    FROM cands c\n    LEFT JOIN winner w\n      ON ((w.parent = c.parent) OR (w.parent IS NULL AND c.parent IS NULL))\n     AND ((w.child  = c.child ) OR (w.child  IS NULL AND c.child  IS NULL))\n    ORDER BY c.parent, c.child\n    \"\"\"\n\n    # If there are restriction filters, wrap the query with INTERSECT\n    # This ensures only resources in the restriction allowlist are returned\n    if restriction_sqls:\n        # Start with the main query, but select only parent/child for the INTERSECT\n        main_query_for_intersect = f\"\"\"\n        WITH\n        cands AS (\n            {candidate_sql}\n        ),\n        rules AS (\n            {union_sql}\n        ),\n        matched AS (\n            SELECT\n                c.parent, c.child,\n                r.allow, r.reason, r.source_plugin,\n                CASE\n                  WHEN r.child  IS NOT NULL THEN 2  -- child-level (most specific)\n                  WHEN r.parent IS NOT NULL THEN 1  -- parent-level\n                  ELSE 0                            -- root/global\n                END AS depth\n            FROM cands c\n            JOIN rules r\n              ON (r.parent IS NULL OR r.parent = c.parent)\n             AND (r.child  IS NULL OR r.child  = c.child)\n        ),\n        ranked AS (\n            SELECT *,\n                   ROW_NUMBER() OVER (\n                     PARTITION BY parent, child\n                     ORDER BY\n                       depth DESC,                          -- specificity first\n                       CASE WHEN allow=0 THEN 0 ELSE 1 END, -- then deny over allow at same depth\n                       source_plugin                        -- stable tie-break\n                   ) AS rn\n            FROM matched\n        ),\n        winner AS (\n            SELECT parent, child,\n                   allow, reason, source_plugin, depth\n            FROM ranked WHERE rn = 1\n        ),\n        permitted_resources AS (\n            SELECT c.parent, c.child\n            FROM cands c\n            LEFT JOIN winner w\n              ON ((w.parent = c.parent) OR (w.parent IS NULL AND c.parent IS NULL))\n             AND ((w.child  = c.child ) OR (w.child  IS NULL AND c.child  IS NULL))\n            WHERE COALESCE(w.allow, CASE WHEN :implicit_deny THEN 0 ELSE NULL END) = 1\n        )\n        SELECT parent, child FROM permitted_resources\n        \"\"\"\n\n        # Build restriction list with INTERSECT (all must match)\n        # Then filter to resources that match hierarchically\n        # Wrap each restriction_sql in a subquery to avoid operator precedence issues\n        # with UNION ALL inside the restriction SQL statements\n        restriction_intersect = \"\\nINTERSECT\\n\".join(\n            f\"SELECT * FROM ({sql})\" for sql in restriction_sqls\n        )\n\n        # Combine: resources allowed by permissions AND in restriction allowlist\n        # Database-level restrictions (parent, NULL) should match all children (parent, *)\n        filtered_resources = f\"\"\"\n        WITH restriction_list AS (\n            {restriction_intersect}\n        ),\n        permitted AS (\n            {main_query_for_intersect}\n        ),\n        filtered AS (\n            SELECT p.parent, p.child\n            FROM permitted p\n            WHERE EXISTS (\n                SELECT 1 FROM restriction_list r\n                WHERE (r.parent = p.parent OR r.parent IS NULL)\n                  AND (r.child = p.child OR r.child IS NULL)\n            )\n        )\n        \"\"\"\n\n        # Now join back to get full results for only the filtered resources\n        sql = f\"\"\"\n        {filtered_resources}\n        , cands AS (\n            {candidate_sql}\n        ),\n        rules AS (\n            {union_sql}\n        ),\n        matched AS (\n            SELECT\n                c.parent, c.child,\n                r.allow, r.reason, r.source_plugin,\n                CASE\n                  WHEN r.child  IS NOT NULL THEN 2  -- child-level (most specific)\n                  WHEN r.parent IS NOT NULL THEN 1  -- parent-level\n                  ELSE 0                            -- root/global\n                END AS depth\n            FROM cands c\n            JOIN rules r\n              ON (r.parent IS NULL OR r.parent = c.parent)\n             AND (r.child  IS NULL OR r.child  = c.child)\n        ),\n        ranked AS (\n            SELECT *,\n                   ROW_NUMBER() OVER (\n                     PARTITION BY parent, child\n                     ORDER BY\n                       depth DESC,                          -- specificity first\n                       CASE WHEN allow=0 THEN 0 ELSE 1 END, -- then deny over allow at same depth\n                       source_plugin                        -- stable tie-break\n                   ) AS rn\n            FROM matched\n        ),\n        winner AS (\n            SELECT parent, child,\n                   allow, reason, source_plugin, depth\n            FROM ranked WHERE rn = 1\n        )\n        SELECT\n          c.parent, c.child,\n          COALESCE(w.allow, CASE WHEN :implicit_deny THEN 0 ELSE NULL END) AS allow,\n          COALESCE(w.reason, CASE WHEN :implicit_deny THEN 'implicit deny' ELSE NULL END) AS reason,\n          w.source_plugin,\n          COALESCE(w.depth, -1) AS depth,\n          :action AS action,\n          CASE\n            WHEN c.parent IS NULL THEN '/'\n            WHEN c.child  IS NULL THEN '/' || c.parent\n            ELSE '/' || c.parent || '/' || c.child\n          END AS resource\n        FROM filtered c\n        LEFT JOIN winner w\n          ON ((w.parent = c.parent) OR (w.parent IS NULL AND c.parent IS NULL))\n         AND ((w.child  = c.child ) OR (w.child  IS NULL AND c.child  IS NULL))\n        ORDER BY c.parent, c.child\n        \"\"\"\n\n    rows_iter: Iterable[sqlite3.Row] = await db.execute(\n        sql,\n        {**all_params, \"implicit_deny\": 1 if implicit_deny else 0},\n    )\n    return [dict(r) for r in rows_iter]\n\n\nasync def resolve_permissions_with_candidates(\n    db,\n    actor: dict | None,\n    plugins: Sequence[Any],\n    candidates: List[Tuple[str, str | None]],\n    action: str,\n    *,\n    implicit_deny: bool = True,\n) -> List[Dict[str, Any]]:\n    \"\"\"\n    Resolve permissions without any external candidate table by embedding\n    the candidates as a UNION of parameterized SELECTs in a CTE.\n\n    candidates: list of (parent, child) where child can be None for parent-scoped actions.\n    actor: actor dict (or None), made available as :actor (JSON), :actor_id, and :action\n    \"\"\"\n    # Build a small CTE for candidates.\n    cand_rows_sql: List[str] = []\n    cand_params: Dict[str, Any] = {}\n    for i, (parent, child) in enumerate(candidates):\n        pkey = f\"cand_p_{i}\"\n        ckey = f\"cand_c_{i}\"\n        cand_params[pkey] = parent\n        cand_params[ckey] = child\n        cand_rows_sql.append(f\"SELECT :{pkey} AS parent, :{ckey} AS child\")\n    candidate_sql = (\n        \"\\nUNION ALL\\n\".join(cand_rows_sql)\n        if cand_rows_sql\n        else \"SELECT NULL AS parent, NULL AS child WHERE 0\"\n    )\n\n    return await resolve_permissions_from_catalog(\n        db,\n        actor,\n        plugins,\n        action,\n        candidate_sql=candidate_sql,\n        candidate_params=cand_params,\n        implicit_deny=implicit_deny,\n    )\n"
  },
  {
    "path": "datasette/utils/shutil_backport.py",
    "content": "\"\"\"\nBackported from Python 3.8.\n\nThis code is licensed under the Python License:\nhttps://github.com/python/cpython/blob/v3.8.3/LICENSE\n\"\"\"\n\nimport os\nfrom shutil import copy, copy2, copystat, Error\n\n\ndef _copytree(\n    entries,\n    src,\n    dst,\n    symlinks,\n    ignore,\n    copy_function,\n    ignore_dangling_symlinks,\n    dirs_exist_ok=False,\n):\n    if ignore is not None:\n        ignored_names = ignore(src, set(os.listdir(src)))\n    else:\n        ignored_names = set()\n\n    os.makedirs(dst, exist_ok=dirs_exist_ok)\n    errors = []\n    use_srcentry = copy_function is copy2 or copy_function is copy\n\n    for srcentry in entries:\n        if srcentry.name in ignored_names:\n            continue\n        srcname = os.path.join(src, srcentry.name)\n        dstname = os.path.join(dst, srcentry.name)\n        srcobj = srcentry if use_srcentry else srcname\n        try:\n            if srcentry.is_symlink():\n                linkto = os.readlink(srcname)\n                if symlinks:\n                    os.symlink(linkto, dstname)\n                    copystat(srcobj, dstname, follow_symlinks=not symlinks)\n                else:\n                    if not os.path.exists(linkto) and ignore_dangling_symlinks:\n                        continue\n                    if srcentry.is_dir():\n                        copytree(\n                            srcobj,\n                            dstname,\n                            symlinks,\n                            ignore,\n                            copy_function,\n                            dirs_exist_ok=dirs_exist_ok,\n                        )\n                    else:\n                        copy_function(srcobj, dstname)\n            elif srcentry.is_dir():\n                copytree(\n                    srcobj,\n                    dstname,\n                    symlinks,\n                    ignore,\n                    copy_function,\n                    dirs_exist_ok=dirs_exist_ok,\n                )\n            else:\n                copy_function(srcentry, dstname)\n        except Error as err:\n            errors.extend(err.args[0])\n        except OSError as why:\n            errors.append((srcname, dstname, str(why)))\n    try:\n        copystat(src, dst)\n    except OSError as why:\n        # Copying file access times may fail on Windows\n        if getattr(why, \"winerror\", None) is None:\n            errors.append((src, dst, str(why)))\n    if errors:\n        raise Error(errors)\n    return dst\n\n\ndef copytree(\n    src,\n    dst,\n    symlinks=False,\n    ignore=None,\n    copy_function=copy2,\n    ignore_dangling_symlinks=False,\n    dirs_exist_ok=False,\n):\n    with os.scandir(src) as entries:\n        return _copytree(\n            entries=entries,\n            src=src,\n            dst=dst,\n            symlinks=symlinks,\n            ignore=ignore,\n            copy_function=copy_function,\n            ignore_dangling_symlinks=ignore_dangling_symlinks,\n            dirs_exist_ok=dirs_exist_ok,\n        )\n"
  },
  {
    "path": "datasette/utils/sqlite.py",
    "content": "using_pysqlite3 = False\ntry:\n    import pysqlite3 as sqlite3\n\n    using_pysqlite3 = True\nexcept ImportError:\n    import sqlite3\n\nif hasattr(sqlite3, \"enable_callback_tracebacks\"):\n    sqlite3.enable_callback_tracebacks(True)\n\n_cached_sqlite_version = None\n\n\ndef sqlite_version():\n    global _cached_sqlite_version\n    if _cached_sqlite_version is None:\n        _cached_sqlite_version = _sqlite_version()\n    return _cached_sqlite_version\n\n\ndef _sqlite_version():\n    return tuple(\n        map(\n            int,\n            sqlite3.connect(\":memory:\")\n            .execute(\"select sqlite_version()\")\n            .fetchone()[0]\n            .split(\".\"),\n        )\n    )\n\n\ndef supports_table_xinfo():\n    return sqlite_version() >= (3, 26, 0)\n\n\ndef supports_generated_columns():\n    return sqlite_version() >= (3, 31, 0)\n"
  },
  {
    "path": "datasette/utils/testing.py",
    "content": "from asgiref.sync import async_to_sync\nfrom urllib.parse import urlencode\nimport json\n\n# These wrapper classes pre-date the introduction of\n# datasette.client and httpx to Datasette. They could\n# be removed if the Datasette tests are modified to\n# call datasette.client directly.\n\n\nclass TestResponse:\n    def __init__(self, httpx_response):\n        self.httpx_response = httpx_response\n\n    @property\n    def status(self):\n        return self.httpx_response.status_code\n\n    # Supports both for test-writing convenience\n    @property\n    def status_code(self):\n        return self.status\n\n    @property\n    def headers(self):\n        return self.httpx_response.headers\n\n    @property\n    def body(self):\n        return self.httpx_response.content\n\n    @property\n    def content(self):\n        return self.body\n\n    @property\n    def cookies(self):\n        return dict(self.httpx_response.cookies)\n\n    @property\n    def json(self):\n        return json.loads(self.text)\n\n    @property\n    def text(self):\n        return self.body.decode(\"utf8\")\n\n\nclass TestClient:\n    max_redirects = 5\n\n    def __init__(self, ds):\n        self.ds = ds\n\n    def actor_cookie(self, actor):\n        return self.ds.sign({\"a\": actor}, \"actor\")\n\n    @async_to_sync\n    async def get(\n        self,\n        path,\n        follow_redirects=False,\n        redirect_count=0,\n        method=\"GET\",\n        params=None,\n        cookies=None,\n        if_none_match=None,\n        headers=None,\n    ):\n        if params:\n            path += \"?\" + urlencode(params, doseq=True)\n        return await self._request(\n            path=path,\n            follow_redirects=follow_redirects,\n            redirect_count=redirect_count,\n            method=method,\n            cookies=cookies,\n            if_none_match=if_none_match,\n            headers=headers,\n        )\n\n    @async_to_sync\n    async def post(\n        self,\n        path,\n        post_data=None,\n        body=None,\n        follow_redirects=False,\n        redirect_count=0,\n        content_type=\"application/x-www-form-urlencoded\",\n        cookies=None,\n        headers=None,\n        csrftoken_from=None,\n    ):\n        cookies = cookies or {}\n        post_data = post_data or {}\n        assert not (post_data and body), \"Provide one or other of body= or post_data=\"\n        # Maybe fetch a csrftoken first\n        if csrftoken_from is not None:\n            assert body is None, \"body= is not compatible with csrftoken_from=\"\n            if csrftoken_from is True:\n                csrftoken_from = path\n            token_response = await self._request(csrftoken_from, cookies=cookies)\n            csrftoken = token_response.cookies[\"ds_csrftoken\"]\n            cookies[\"ds_csrftoken\"] = csrftoken\n            post_data[\"csrftoken\"] = csrftoken\n        if post_data:\n            body = urlencode(post_data, doseq=True)\n        return await self._request(\n            path=path,\n            follow_redirects=follow_redirects,\n            redirect_count=redirect_count,\n            method=\"POST\",\n            cookies=cookies,\n            headers=headers,\n            post_body=body,\n            content_type=content_type,\n        )\n\n    @async_to_sync\n    async def request(\n        self,\n        path,\n        follow_redirects=True,\n        redirect_count=0,\n        method=\"GET\",\n        cookies=None,\n        headers=None,\n        post_body=None,\n        content_type=None,\n        if_none_match=None,\n    ):\n        return await self._request(\n            path,\n            follow_redirects=follow_redirects,\n            redirect_count=redirect_count,\n            method=method,\n            cookies=cookies,\n            headers=headers,\n            post_body=post_body,\n            content_type=content_type,\n            if_none_match=if_none_match,\n        )\n\n    async def _request(\n        self,\n        path,\n        follow_redirects=True,\n        redirect_count=0,\n        method=\"GET\",\n        cookies=None,\n        headers=None,\n        post_body=None,\n        content_type=None,\n        if_none_match=None,\n    ):\n        await self.ds.invoke_startup()\n        headers = headers or {}\n        if content_type:\n            headers[\"content-type\"] = content_type\n        if if_none_match:\n            headers[\"if-none-match\"] = if_none_match\n        httpx_response = await self.ds.client.request(\n            method,\n            path,\n            follow_redirects=follow_redirects,\n            avoid_path_rewrites=True,\n            cookies=cookies,\n            headers=headers,\n            content=post_body,\n        )\n        response = TestResponse(httpx_response)\n        if follow_redirects and response.status in (301, 302):\n            assert (\n                redirect_count < self.max_redirects\n            ), f\"Redirected {redirect_count} times, max_redirects={self.max_redirects}\"\n            location = response.headers[\"Location\"]\n            return await self._request(\n                location, follow_redirects=True, redirect_count=redirect_count + 1\n            )\n        return response\n"
  },
  {
    "path": "datasette/version.py",
    "content": "__version__ = \"1.0a26\"\n__version_info__ = tuple(__version__.split(\".\"))\n"
  },
  {
    "path": "datasette/views/__init__.py",
    "content": "class Context:\n    \"Base class for all documented contexts\"\n"
  },
  {
    "path": "datasette/views/base.py",
    "content": "import asyncio\nimport csv\nimport hashlib\nimport sys\nimport textwrap\nimport time\nimport urllib\nfrom markupsafe import escape\n\n\nfrom datasette.database import QueryInterrupted\nfrom datasette.utils.asgi import Request\nfrom datasette.utils import (\n    add_cors_headers,\n    await_me_maybe,\n    EscapeHtmlWriter,\n    InvalidSql,\n    LimitedWriter,\n    call_with_supported_arguments,\n    path_from_row_pks,\n    path_with_added_args,\n    path_with_removed_args,\n    path_with_format,\n    sqlite3,\n)\nfrom datasette.utils.asgi import (\n    AsgiStream,\n    NotFound,\n    Response,\n    BadRequest,\n)\n\n\nclass DatasetteError(Exception):\n    def __init__(\n        self,\n        message,\n        title=None,\n        error_dict=None,\n        status=500,\n        template=None,\n        message_is_html=False,\n    ):\n        self.message = message\n        self.title = title\n        self.error_dict = error_dict or {}\n        self.status = status\n        self.message_is_html = message_is_html\n\n\nclass View:\n    async def head(self, request, datasette):\n        if not hasattr(self, \"get\"):\n            return await self.method_not_allowed(request)\n        response = await self.get(request, datasette)\n        response.body = \"\"\n        return response\n\n    async def method_not_allowed(self, request):\n        if (\n            request.path.endswith(\".json\")\n            or request.headers.get(\"content-type\") == \"application/json\"\n        ):\n            response = Response.json(\n                {\"ok\": False, \"error\": \"Method not allowed\"}, status=405\n            )\n        else:\n            response = Response.text(\"Method not allowed\", status=405)\n        return response\n\n    async def options(self, request, datasette):\n        response = Response.text(\"ok\")\n        response.headers[\"allow\"] = \", \".join(\n            method.upper()\n            for method in (\"head\", \"get\", \"post\", \"put\", \"patch\", \"delete\")\n            if hasattr(self, method)\n        )\n        return response\n\n    async def __call__(self, request, datasette):\n        try:\n            handler = getattr(self, request.method.lower())\n        except AttributeError:\n            return await self.method_not_allowed(request)\n        return await handler(request, datasette)\n\n\nclass BaseView:\n    ds = None\n    has_json_alternate = True\n\n    def __init__(self, datasette):\n        self.ds = datasette\n\n    async def head(self, *args, **kwargs):\n        response = await self.get(*args, **kwargs)\n        response.body = b\"\"\n        return response\n\n    async def method_not_allowed(self, request):\n        if (\n            request.path.endswith(\".json\")\n            or request.headers.get(\"content-type\") == \"application/json\"\n        ):\n            response = Response.json(\n                {\"ok\": False, \"error\": \"Method not allowed\"}, status=405\n            )\n        else:\n            response = Response.text(\"Method not allowed\", status=405)\n        return response\n\n    async def options(self, request, *args, **kwargs):\n        return Response.text(\"ok\")\n\n    async def get(self, request, *args, **kwargs):\n        return await self.method_not_allowed(request)\n\n    async def post(self, request, *args, **kwargs):\n        return await self.method_not_allowed(request)\n\n    async def put(self, request, *args, **kwargs):\n        return await self.method_not_allowed(request)\n\n    async def patch(self, request, *args, **kwargs):\n        return await self.method_not_allowed(request)\n\n    async def delete(self, request, *args, **kwargs):\n        return await self.method_not_allowed(request)\n\n    async def dispatch_request(self, request):\n        if self.ds:\n            await self.ds.refresh_schemas()\n        handler = getattr(self, request.method.lower(), None)\n        response = await handler(request)\n        if self.ds.cors:\n            add_cors_headers(response.headers)\n        return response\n\n    async def render(self, templates, request, context=None):\n        context = context or {}\n        environment = self.ds.get_jinja_environment(request)\n        template = environment.select_template(templates)\n        template_context = {\n            **context,\n            **{\n                \"select_templates\": [\n                    f\"{'*' if template_name == template.name else ''}{template_name}\"\n                    for template_name in templates\n                ],\n            },\n        }\n        headers = {}\n        if self.has_json_alternate:\n            alternate_url_json = self.ds.absolute_url(\n                request,\n                self.ds.urls.path(path_with_format(request=request, format=\"json\")),\n            )\n            template_context[\"alternate_url_json\"] = alternate_url_json\n            headers.update(\n                {\n                    \"Link\": '<{}>; rel=\"alternate\"; type=\"application/json+datasette\"'.format(\n                        alternate_url_json\n                    )\n                }\n            )\n        return Response.html(\n            await self.ds.render_template(\n                template,\n                template_context,\n                request=request,\n                view_name=self.name,\n            ),\n            headers=headers,\n        )\n\n    @classmethod\n    def as_view(cls, *class_args, **class_kwargs):\n        async def view(request, send):\n            self = view.view_class(*class_args, **class_kwargs)\n            return await self.dispatch_request(request)\n\n        view.view_class = cls\n        view.__doc__ = cls.__doc__\n        view.__module__ = cls.__module__\n        view.__name__ = cls.__name__\n        return view\n\n\nclass DataView(BaseView):\n    name = \"\"\n\n    def redirect(self, request, path, forward_querystring=True, remove_args=None):\n        if request.query_string and \"?\" not in path and forward_querystring:\n            path = f\"{path}?{request.query_string}\"\n        if remove_args:\n            path = path_with_removed_args(request, remove_args, path=path)\n        r = Response.redirect(path)\n        r.headers[\"Link\"] = f\"<{path}>; rel=preload\"\n        if self.ds.cors:\n            add_cors_headers(r.headers)\n        return r\n\n    async def data(self, request):\n        raise NotImplementedError\n\n    async def as_csv(self, request, database):\n        return await stream_csv(self.ds, self.data, request, database)\n\n    async def get(self, request):\n        db = await self.ds.resolve_database(request)\n        database = db.name\n        database_route = db.route\n\n        _format = request.url_vars[\"format\"]\n        data_kwargs = {}\n\n        if _format == \"csv\":\n            return await self.as_csv(request, database_route)\n\n        if _format is None:\n            # HTML views default to expanding all foreign key labels\n            data_kwargs[\"default_labels\"] = True\n\n        extra_template_data = {}\n        start = time.perf_counter()\n        status_code = None\n        templates = []\n        try:\n            response_or_template_contexts = await self.data(request, **data_kwargs)\n            if isinstance(response_or_template_contexts, Response):\n                return response_or_template_contexts\n            # If it has four items, it includes an HTTP status code\n            if len(response_or_template_contexts) == 4:\n                (\n                    data,\n                    extra_template_data,\n                    templates,\n                    status_code,\n                ) = response_or_template_contexts\n            else:\n                data, extra_template_data, templates = response_or_template_contexts\n        except QueryInterrupted as ex:\n            raise DatasetteError(\n                textwrap.dedent(\"\"\"\n                
    SQL query took too long. The time limit is controlled by the\n                sql_time_limit_ms\n                configuration option.
    \n                \n                \n            \"\"\".format(escape(ex.sql))).strip(),\n                title=\"SQL Interrupted\",\n                status=400,\n                message_is_html=True,\n            )\n        except (sqlite3.OperationalError, InvalidSql) as e:\n            raise DatasetteError(str(e), title=\"Invalid SQL\", status=400)\n\n        except sqlite3.OperationalError as e:\n            raise DatasetteError(str(e))\n\n        except DatasetteError:\n            raise\n\n        end = time.perf_counter()\n        data[\"query_ms\"] = (end - start) * 1000\n\n        # Special case for .jsono extension - redirect to _shape=objects\n        if _format == \"jsono\":\n            return self.redirect(\n                request,\n                path_with_added_args(\n                    request,\n                    {\"_shape\": \"objects\"},\n                    path=request.path.rsplit(\".jsono\", 1)[0] + \".json\",\n                ),\n                forward_querystring=False,\n            )\n\n        if _format in self.ds.renderers.keys():\n            # Dispatch request to the correct output format renderer\n            # (CSV is not handled here due to streaming)\n            result = call_with_supported_arguments(\n                self.ds.renderers[_format][0],\n                datasette=self.ds,\n                columns=data.get(\"columns\") or [],\n                rows=data.get(\"rows\") or [],\n                sql=data.get(\"query\", {}).get(\"sql\", None),\n                query_name=data.get(\"query_name\"),\n                database=database,\n                table=data.get(\"table\"),\n                request=request,\n                view_name=self.name,\n                truncated=False,  # TODO: support this\n                error=data.get(\"error\"),\n                # These will be deprecated in Datasette 1.0:\n                args=request.args,\n                data=data,\n            )\n            if asyncio.iscoroutine(result):\n                result = await result\n            if result is None:\n                raise NotFound(\"No data\")\n            if isinstance(result, dict):\n                r = Response(\n                    body=result.get(\"body\"),\n                    status=result.get(\"status_code\", status_code or 200),\n                    content_type=result.get(\"content_type\", \"text/plain\"),\n                    headers=result.get(\"headers\"),\n                )\n            elif isinstance(result, Response):\n                r = result\n                if status_code is not None:\n                    # Over-ride the status code\n                    r.status = status_code\n            else:\n                assert False, f\"{result} should be dict or Response\"\n        else:\n            extras = {}\n            if callable(extra_template_data):\n                extras = extra_template_data()\n                if asyncio.iscoroutine(extras):\n                    extras = await extras\n            else:\n                extras = extra_template_data\n            url_labels_extra = {}\n            if data.get(\"expandable_columns\"):\n                url_labels_extra = {\"_labels\": \"on\"}\n\n            renderers = {}\n            for key, (_, can_render) in self.ds.renderers.items():\n                it_can_render = call_with_supported_arguments(\n                    can_render,\n                    datasette=self.ds,\n                    columns=data.get(\"columns\") or [],\n                    rows=data.get(\"rows\") or [],\n                    sql=data.get(\"query\", {}).get(\"sql\", None),\n                    query_name=data.get(\"query_name\"),\n                    database=database,\n                    table=data.get(\"table\"),\n                    request=request,\n                    view_name=self.name,\n                )\n                it_can_render = await await_me_maybe(it_can_render)\n                if it_can_render:\n                    renderers[key] = self.ds.urls.path(\n                        path_with_format(\n                            request=request, format=key, extra_qs={**url_labels_extra}\n                        )\n                    )\n\n            url_csv_args = {\"_size\": \"max\", **url_labels_extra}\n            url_csv = self.ds.urls.path(\n                path_with_format(request=request, format=\"csv\", extra_qs=url_csv_args)\n            )\n            url_csv_path = url_csv.split(\"?\")[0]\n            context = {\n                **data,\n                **extras,\n                **{\n                    \"renderers\": renderers,\n                    \"url_csv\": url_csv,\n                    \"url_csv_path\": url_csv_path,\n                    \"url_csv_hidden_args\": [\n                        (key, value)\n                        for key, value in urllib.parse.parse_qsl(request.query_string)\n                        if key not in (\"_labels\", \"_facet\", \"_size\")\n                    ]\n                    + [(\"_size\", \"max\")],\n                    \"settings\": self.ds.settings_dict(),\n                },\n            }\n            if \"metadata\" not in context:\n                context[\"metadata\"] = await self.ds.get_instance_metadata()\n            r = await self.render(templates, request=request, context=context)\n            if status_code is not None:\n                r.status = status_code\n\n        ttl = request.args.get(\"_ttl\", None)\n        if ttl is None or not ttl.isdigit():\n            ttl = self.ds.setting(\"default_cache_ttl\")\n\n        return self.set_response_headers(r, ttl)\n\n    def set_response_headers(self, response, ttl):\n        # Set far-future cache expiry\n        if self.ds.cache_headers and response.status == 200:\n            ttl = int(ttl)\n            if ttl == 0:\n                ttl_header = \"no-cache\"\n            else:\n                ttl_header = f\"max-age={ttl}\"\n            response.headers[\"Cache-Control\"] = ttl_header\n        response.headers[\"Referrer-Policy\"] = \"no-referrer\"\n        if self.ds.cors:\n            add_cors_headers(response.headers)\n        return response\n\n\ndef _error(messages, status=400):\n    return Response.json({\"ok\": False, \"errors\": messages}, status=status)\n\n\nasync def stream_csv(datasette, fetch_data, request, database):\n    kwargs = {}\n    stream = request.args.get(\"_stream\")\n    # Do not calculate facets or counts:\n    extra_parameters = [\n        \"{}=1\".format(key)\n        for key in (\"_nofacet\", \"_nocount\")\n        if not request.args.get(key)\n    ]\n    if extra_parameters:\n        # Replace request object with a new one with modified scope\n        if not request.query_string:\n            new_query_string = \"&\".join(extra_parameters)\n        else:\n            new_query_string = request.query_string + \"&\" + \"&\".join(extra_parameters)\n        new_scope = dict(request.scope, query_string=new_query_string.encode(\"latin-1\"))\n        receive = request.receive\n        request = Request(new_scope, receive)\n    if stream:\n        # Some quick soundness checks\n        if not datasette.setting(\"allow_csv_stream\"):\n            raise BadRequest(\"CSV streaming is disabled\")\n        if request.args.get(\"_next\"):\n            raise BadRequest(\"_next not allowed for CSV streaming\")\n        kwargs[\"_size\"] = \"max\"\n    # Fetch the first page\n    try:\n        response_or_template_contexts = await fetch_data(request)\n        if isinstance(response_or_template_contexts, Response):\n            return response_or_template_contexts\n        elif len(response_or_template_contexts) == 4:\n            data, _, _, _ = response_or_template_contexts\n        else:\n            data, _, _ = response_or_template_contexts\n    except (sqlite3.OperationalError, InvalidSql) as e:\n        raise DatasetteError(str(e), title=\"Invalid SQL\", status=400)\n\n    except sqlite3.OperationalError as e:\n        raise DatasetteError(str(e))\n\n    except DatasetteError:\n        raise\n\n    # Convert rows and columns to CSV\n    headings = data[\"columns\"]\n    # if there are expanded_columns we need to add additional headings\n    expanded_columns = set(data.get(\"expanded_columns\") or [])\n    if expanded_columns:\n        headings = []\n        for column in data[\"columns\"]:\n            headings.append(column)\n            if column in expanded_columns:\n                headings.append(f\"{column}_label\")\n\n    content_type = \"text/plain; charset=utf-8\"\n    preamble = \"\"\n    postamble = \"\"\n\n    trace = request.args.get(\"_trace\")\n    if trace:\n        content_type = \"text/html; charset=utf-8\"\n        preamble = (\n            \"CSV debug\"\n            '\"\n\n    async def stream_fn(r):\n        nonlocal data, trace\n        limited_writer = LimitedWriter(r, datasette.setting(\"max_csv_mb\"))\n        if trace:\n            await limited_writer.write(preamble)\n            writer = csv.writer(EscapeHtmlWriter(limited_writer))\n        else:\n            writer = csv.writer(limited_writer)\n        first = True\n        next = None\n        while first or (next and stream):\n            try:\n                kwargs = {}\n                if next:\n                    kwargs[\"_next\"] = next\n                if not first:\n                    data, _, _ = await fetch_data(request, **kwargs)\n                if first:\n                    if request.args.get(\"_header\") != \"off\":\n                        await writer.writerow(headings)\n                    first = False\n                next = data.get(\"next\")\n                for row in data[\"rows\"]:\n                    if any(isinstance(r, bytes) for r in row):\n                        new_row = []\n                        for column, cell in zip(headings, row):\n                            if isinstance(cell, bytes):\n                                # If this is a table page, use .urls.row_blob()\n                                if data.get(\"table\"):\n                                    pks = data.get(\"primary_keys\") or []\n                                    cell = datasette.absolute_url(\n                                        request,\n                                        datasette.urls.row_blob(\n                                            database,\n                                            data[\"table\"],\n                                            path_from_row_pks(row, pks, not pks),\n                                            column,\n                                        ),\n                                    )\n                                else:\n                                    # Otherwise generate URL for this query\n                                    url = datasette.absolute_url(\n                                        request,\n                                        path_with_format(\n                                            request=request,\n                                            format=\"blob\",\n                                            extra_qs={\n                                                \"_blob_column\": column,\n                                                \"_blob_hash\": hashlib.sha256(\n                                                    cell\n                                                ).hexdigest(),\n                                            },\n                                            replace_format=\"csv\",\n                                        ),\n                                    )\n                                    cell = url.replace(\"&_nocount=1\", \"\").replace(\n                                        \"&_nofacet=1\", \"\"\n                                    )\n                            new_row.append(cell)\n                        row = new_row\n                    if not expanded_columns:\n                        # Simple path\n                        await writer.writerow(row)\n                    else:\n                        # Look for {\"value\": \"label\": } dicts and expand\n                        new_row = []\n                        for heading, cell in zip(data[\"columns\"], row):\n                            if heading in expanded_columns:\n                                if cell is None:\n                                    new_row.extend((\"\", \"\"))\n                                else:\n                                    if not isinstance(cell, dict):\n                                        new_row.extend((cell, \"\"))\n                                    else:\n                                        new_row.append(cell[\"value\"])\n                                        new_row.append(cell[\"label\"])\n                            else:\n                                new_row.append(cell)\n                        await writer.writerow(new_row)\n            except Exception as ex:\n                sys.stderr.write(\"Caught this error: {}\\n\".format(ex))\n                sys.stderr.flush()\n                await r.write(str(ex))\n                return\n        await limited_writer.write(postamble)\n\n    headers = {}\n    if datasette.cors:\n        add_cors_headers(headers)\n    if request.args.get(\"_dl\", None):\n        if not trace:\n            content_type = \"text/csv; charset=utf-8\"\n        disposition = 'attachment; filename=\"{}.csv\"'.format(\n            request.url_vars.get(\"table\", database)\n        )\n        headers[\"content-disposition\"] = disposition\n\n    return AsgiStream(stream_fn, headers=headers, content_type=content_type)\n"
  },
  {
    "path": "datasette/views/database.py",
    "content": "from dataclasses import dataclass, field\nfrom urllib.parse import parse_qsl, urlencode\nimport asyncio\nimport hashlib\nimport itertools\nimport json\nimport markupsafe\nimport os\nimport re\nimport sqlite_utils\nimport textwrap\n\nfrom datasette.events import AlterTableEvent, CreateTableEvent, InsertRowsEvent\nfrom datasette.database import QueryInterrupted\nfrom datasette.resources import DatabaseResource, QueryResource\nfrom datasette.utils import (\n    add_cors_headers,\n    await_me_maybe,\n    call_with_supported_arguments,\n    named_parameters as derive_named_parameters,\n    format_bytes,\n    make_slot_function,\n    tilde_decode,\n    to_css_class,\n    validate_sql_select,\n    is_url,\n    path_with_added_args,\n    path_with_format,\n    path_with_removed_args,\n    sqlite3,\n    truncate_url,\n    InvalidSql,\n)\nfrom datasette.utils.asgi import AsgiFileDownload, NotFound, Response, Forbidden\nfrom datasette.plugins import pm\n\nfrom .base import BaseView, DatasetteError, View, _error, stream_csv\nfrom . import Context\n\n\nclass DatabaseView(View):\n    async def get(self, request, datasette):\n        format_ = request.url_vars.get(\"format\") or \"html\"\n\n        await datasette.refresh_schemas()\n\n        db = await datasette.resolve_database(request)\n        database = db.name\n\n        visible, private = await datasette.check_visibility(\n            request.actor,\n            action=\"view-database\",\n            resource=DatabaseResource(database=database),\n        )\n        if not visible:\n            raise Forbidden(\"You do not have permission to view this database\")\n\n        sql = (request.args.get(\"sql\") or \"\").strip()\n        if sql:\n            redirect_url = \"/\" + request.url_vars.get(\"database\") + \"/-/query\"\n            if request.url_vars.get(\"format\"):\n                redirect_url += \".\" + request.url_vars.get(\"format\")\n            redirect_url += \"?\" + request.query_string\n            return Response.redirect(redirect_url)\n            return await QueryView()(request, datasette)\n\n        if format_ not in (\"html\", \"json\"):\n            raise NotFound(\"Invalid format: {}\".format(format_))\n\n        metadata = await datasette.get_database_metadata(database)\n\n        # Get all tables/views this actor can see in bulk with private flag\n        allowed_tables_page = await datasette.allowed_resources(\n            \"view-table\",\n            request.actor,\n            parent=database,\n            include_is_private=True,\n            limit=1000,\n        )\n        # Create lookup dict for quick access\n        allowed_dict = {r.child: r for r in allowed_tables_page.resources}\n\n        # Filter to just views\n        view_names_set = set(await db.view_names())\n        sql_views = [\n            {\"name\": name, \"private\": allowed_dict[name].private}\n            for name in allowed_dict\n            if name in view_names_set\n        ]\n\n        tables = await get_tables(datasette, request, db, allowed_dict)\n\n        # Get allowed queries using the new permission system\n        allowed_query_page = await datasette.allowed_resources(\n            \"view-query\",\n            request.actor,\n            parent=database,\n            include_is_private=True,\n            limit=1000,\n        )\n\n        # Build canned_queries list by looking up each allowed query\n        all_queries = await datasette.get_canned_queries(database, request.actor)\n        canned_queries = []\n        for query_resource in allowed_query_page.resources:\n            query_name = query_resource.child\n            if query_name in all_queries:\n                canned_queries.append(\n                    dict(all_queries[query_name], private=query_resource.private)\n                )\n\n        async def database_actions():\n            links = []\n            for hook in pm.hook.database_actions(\n                datasette=datasette,\n                database=database,\n                actor=request.actor,\n                request=request,\n            ):\n                extra_links = await await_me_maybe(hook)\n                if extra_links:\n                    links.extend(extra_links)\n            return links\n\n        attached_databases = [d.name for d in await db.attached_databases()]\n\n        allow_execute_sql = await datasette.allowed(\n            action=\"execute-sql\",\n            resource=DatabaseResource(database=database),\n            actor=request.actor,\n        )\n        json_data = {\n            \"database\": database,\n            \"private\": private,\n            \"path\": datasette.urls.database(database),\n            \"size\": db.size,\n            \"tables\": tables,\n            \"hidden_count\": len([t for t in tables if t[\"hidden\"]]),\n            \"views\": sql_views,\n            \"queries\": canned_queries,\n            \"allow_execute_sql\": allow_execute_sql,\n            \"table_columns\": (\n                await _table_columns(datasette, database) if allow_execute_sql else {}\n            ),\n            \"metadata\": await datasette.get_database_metadata(database),\n        }\n\n        if format_ == \"json\":\n            response = Response.json(json_data)\n            if datasette.cors:\n                add_cors_headers(response.headers)\n            return response\n\n        assert format_ == \"html\"\n        alternate_url_json = datasette.absolute_url(\n            request,\n            datasette.urls.path(path_with_format(request=request, format=\"json\")),\n        )\n        templates = (f\"database-{to_css_class(database)}.html\", \"database.html\")\n        environment = datasette.get_jinja_environment(request)\n        template = environment.select_template(templates)\n        return Response.html(\n            await datasette.render_template(\n                templates,\n                DatabaseContext(\n                    database=database,\n                    private=private,\n                    path=datasette.urls.database(database),\n                    size=db.size,\n                    tables=tables,\n                    hidden_count=len([t for t in tables if t[\"hidden\"]]),\n                    views=sql_views,\n                    queries=canned_queries,\n                    allow_execute_sql=allow_execute_sql,\n                    table_columns=(\n                        await _table_columns(datasette, database)\n                        if allow_execute_sql\n                        else {}\n                    ),\n                    metadata=metadata,\n                    database_color=db.color,\n                    database_actions=database_actions,\n                    show_hidden=request.args.get(\"_show_hidden\"),\n                    editable=True,\n                    count_limit=db.count_limit,\n                    allow_download=datasette.setting(\"allow_download\")\n                    and not db.is_mutable\n                    and not db.is_memory,\n                    attached_databases=attached_databases,\n                    alternate_url_json=alternate_url_json,\n                    select_templates=[\n                        f\"{'*' if template_name == template.name else ''}{template_name}\"\n                        for template_name in templates\n                    ],\n                    top_database=make_slot_function(\n                        \"top_database\", datasette, request, database=database\n                    ),\n                ),\n                request=request,\n                view_name=\"database\",\n            ),\n            headers={\n                \"Link\": '<{}>; rel=\"alternate\"; type=\"application/json+datasette\"'.format(\n                    alternate_url_json\n                )\n            },\n        )\n\n\n@dataclass\nclass DatabaseContext(Context):\n    database: str = field(metadata={\"help\": \"The name of the database\"})\n    private: bool = field(\n        metadata={\"help\": \"Boolean indicating if this is a private database\"}\n    )\n    path: str = field(metadata={\"help\": \"The URL path to this database\"})\n    size: int = field(metadata={\"help\": \"The size of the database in bytes\"})\n    tables: list = field(metadata={\"help\": \"List of table objects in the database\"})\n    hidden_count: int = field(metadata={\"help\": \"Count of hidden tables\"})\n    views: list = field(metadata={\"help\": \"List of view objects in the database\"})\n    queries: list = field(metadata={\"help\": \"List of canned query objects\"})\n    allow_execute_sql: bool = field(\n        metadata={\"help\": \"Boolean indicating if custom SQL can be executed\"}\n    )\n    table_columns: dict = field(\n        metadata={\"help\": \"Dictionary mapping table names to their column lists\"}\n    )\n    metadata: dict = field(metadata={\"help\": \"Metadata for the database\"})\n    database_color: str = field(metadata={\"help\": \"The color assigned to the database\"})\n    database_actions: callable = field(\n        metadata={\n            \"help\": \"Callable returning list of action links for the database menu\"\n        }\n    )\n    show_hidden: str = field(metadata={\"help\": \"Value of _show_hidden query parameter\"})\n    editable: bool = field(\n        metadata={\"help\": \"Boolean indicating if the database is editable\"}\n    )\n    count_limit: int = field(metadata={\"help\": \"The maximum number of rows to count\"})\n    allow_download: bool = field(\n        metadata={\"help\": \"Boolean indicating if database download is allowed\"}\n    )\n    attached_databases: list = field(\n        metadata={\"help\": \"List of names of attached databases\"}\n    )\n    alternate_url_json: str = field(\n        metadata={\"help\": \"URL for the alternate JSON version of this page\"}\n    )\n    select_templates: list = field(\n        metadata={\n            \"help\": \"List of templates that were considered for rendering this page\"\n        }\n    )\n    top_database: callable = field(\n        metadata={\"help\": \"Callable to render the top_database slot\"}\n    )\n\n\n@dataclass\nclass QueryContext(Context):\n    database: str = field(metadata={\"help\": \"The name of the database being queried\"})\n    database_color: str = field(metadata={\"help\": \"The color of the database\"})\n    query: dict = field(\n        metadata={\"help\": \"The SQL query object containing the `sql` string\"}\n    )\n    canned_query: str = field(\n        metadata={\"help\": \"The name of the canned query if this is a canned query\"}\n    )\n    private: bool = field(\n        metadata={\"help\": \"Boolean indicating if this is a private database\"}\n    )\n    # urls: dict = field(\n    #     metadata={\"help\": \"Object containing URL helpers like `database()`\"}\n    # )\n    canned_query_write: bool = field(\n        metadata={\n            \"help\": \"Boolean indicating if this is a canned query that allows writes\"\n        }\n    )\n    metadata: dict = field(\n        metadata={\"help\": \"Metadata about the database or the canned query\"}\n    )\n    db_is_immutable: bool = field(\n        metadata={\"help\": \"Boolean indicating if this database is immutable\"}\n    )\n    error: str = field(metadata={\"help\": \"Any query error message\"})\n    hide_sql: bool = field(\n        metadata={\"help\": \"Boolean indicating if the SQL should be hidden\"}\n    )\n    show_hide_link: str = field(\n        metadata={\"help\": \"The URL to toggle showing/hiding the SQL\"}\n    )\n    show_hide_text: str = field(\n        metadata={\"help\": \"The text for the show/hide SQL link\"}\n    )\n    editable: bool = field(\n        metadata={\"help\": \"Boolean indicating if the SQL can be edited\"}\n    )\n    allow_execute_sql: bool = field(\n        metadata={\"help\": \"Boolean indicating if custom SQL can be executed\"}\n    )\n    tables: list = field(metadata={\"help\": \"List of table objects in the database\"})\n    named_parameter_values: dict = field(\n        metadata={\"help\": \"Dictionary of parameter names/values\"}\n    )\n    edit_sql_url: str = field(\n        metadata={\"help\": \"URL to edit the SQL for a canned query\"}\n    )\n    display_rows: list = field(metadata={\"help\": \"List of result rows to display\"})\n    columns: list = field(metadata={\"help\": \"List of column names\"})\n    renderers: dict = field(metadata={\"help\": \"Dictionary of renderer name to URL\"})\n    url_csv: str = field(metadata={\"help\": \"URL for CSV export\"})\n    show_hide_hidden: str = field(\n        metadata={\"help\": \"Hidden input field for the _show_sql parameter\"}\n    )\n    table_columns: dict = field(\n        metadata={\"help\": \"Dictionary of table name to list of column names\"}\n    )\n    alternate_url_json: str = field(\n        metadata={\"help\": \"URL for alternate JSON version of this page\"}\n    )\n    # TODO: refactor this to somewhere else, probably ds.render_template()\n    select_templates: list = field(\n        metadata={\n            \"help\": \"List of templates that were considered for rendering this page\"\n        }\n    )\n    top_query: callable = field(\n        metadata={\"help\": \"Callable to render the top_query slot\"}\n    )\n    top_canned_query: callable = field(\n        metadata={\"help\": \"Callable to render the top_canned_query slot\"}\n    )\n    query_actions: callable = field(\n        metadata={\n            \"help\": \"Callable returning a list of links for the query action menu\"\n        }\n    )\n\n\nasync def get_tables(datasette, request, db, allowed_dict):\n    \"\"\"\n    Get list of tables with metadata for the database view.\n\n    Args:\n        datasette: The Datasette instance\n        request: The current request\n        db: The database\n        allowed_dict: Dict mapping table name -> Resource object with .private attribute\n    \"\"\"\n    tables = []\n    table_counts = await db.table_counts(100)\n    hidden_table_names = set(await db.hidden_table_names())\n    all_foreign_keys = await db.get_all_foreign_keys()\n\n    for table in table_counts:\n        if table not in allowed_dict:\n            continue\n\n        table_columns = await db.table_columns(table)\n        tables.append(\n            {\n                \"name\": table,\n                \"columns\": table_columns,\n                \"primary_keys\": await db.primary_keys(table),\n                \"count\": table_counts[table],\n                \"hidden\": table in hidden_table_names,\n                \"fts_table\": await db.fts_table(table),\n                \"foreign_keys\": all_foreign_keys[table],\n                \"private\": allowed_dict[table].private,\n            }\n        )\n    tables.sort(key=lambda t: (t[\"hidden\"], t[\"name\"]))\n    return tables\n\n\nasync def database_download(request, datasette):\n    from datasette.resources import DatabaseResource\n\n    database = tilde_decode(request.url_vars[\"database\"])\n    await datasette.ensure_permission(\n        action=\"view-database-download\",\n        resource=DatabaseResource(database=database),\n        actor=request.actor,\n    )\n    try:\n        db = datasette.get_database(route=database)\n    except KeyError:\n        raise DatasetteError(\"Invalid database\", status=404)\n\n    if db.is_memory:\n        raise DatasetteError(\"Cannot download in-memory databases\", status=404)\n    if not datasette.setting(\"allow_download\") or db.is_mutable:\n        raise Forbidden(\"Database download is forbidden\")\n    if not db.path:\n        raise DatasetteError(\"Cannot download database\", status=404)\n    filepath = db.path\n    headers = {}\n    if datasette.cors:\n        add_cors_headers(headers)\n    if db.hash:\n        etag = '\"{}\"'.format(db.hash)\n        headers[\"Etag\"] = etag\n        # Has user seen this already?\n        if_none_match = request.headers.get(\"if-none-match\")\n        if if_none_match and if_none_match == etag:\n            return Response(\"\", status=304)\n    headers[\"Transfer-Encoding\"] = \"chunked\"\n    return AsgiFileDownload(\n        filepath,\n        filename=os.path.basename(filepath),\n        content_type=\"application/octet-stream\",\n        headers=headers,\n    )\n\n\nclass QueryView(View):\n    async def post(self, request, datasette):\n        from datasette.app import TableNotFound\n\n        db = await datasette.resolve_database(request)\n\n        # We must be a canned query\n        table_found = False\n        try:\n            await datasette.resolve_table(request)\n            table_found = True\n        except TableNotFound as table_not_found:\n            canned_query = await datasette.get_canned_query(\n                table_not_found.database_name, table_not_found.table, request.actor\n            )\n            if canned_query is None:\n                raise\n        if table_found:\n            # That should not have happened\n            raise DatasetteError(\"Unexpected table found on POST\", status=404)\n\n        # If database is immutable, return an error\n        if not db.is_mutable:\n            raise Forbidden(\"Database is immutable\")\n\n        # Process the POST\n        body = await request.post_body()\n        body = body.decode(\"utf-8\").strip()\n        if body.startswith(\"{\") and body.endswith(\"}\"):\n            params = json.loads(body)\n            # But we want key=value strings\n            for key, value in params.items():\n                params[key] = str(value)\n        else:\n            params = dict(parse_qsl(body, keep_blank_values=True))\n\n        # Don't ever send csrftoken as a SQL parameter\n        params.pop(\"csrftoken\", None)\n\n        # Should we return JSON?\n        should_return_json = (\n            request.headers.get(\"accept\") == \"application/json\"\n            or request.args.get(\"_json\")\n            or params.get(\"_json\")\n        )\n        params_for_query = MagicParameters(\n            canned_query[\"sql\"], params, request, datasette\n        )\n        await params_for_query.execute_params()\n        ok = None\n        redirect_url = None\n        try:\n            cursor = await db.execute_write(\n                canned_query[\"sql\"], params_for_query, request=request\n            )\n            # success message can come from on_success_message or on_success_message_sql\n            message = None\n            message_type = datasette.INFO\n            on_success_message_sql = canned_query.get(\"on_success_message_sql\")\n            if on_success_message_sql:\n                try:\n                    message_result = (\n                        await db.execute(on_success_message_sql, params_for_query)\n                    ).first()\n                    if message_result:\n                        message = message_result[0]\n                except Exception as ex:\n                    message = \"Error running on_success_message_sql: {}\".format(ex)\n                    message_type = datasette.ERROR\n            if not message:\n                message = canned_query.get(\n                    \"on_success_message\"\n                ) or \"Query executed, {} row{} affected\".format(\n                    cursor.rowcount, \"\" if cursor.rowcount == 1 else \"s\"\n                )\n\n            redirect_url = canned_query.get(\"on_success_redirect\")\n            ok = True\n        except Exception as ex:\n            message = canned_query.get(\"on_error_message\") or str(ex)\n            message_type = datasette.ERROR\n            redirect_url = canned_query.get(\"on_error_redirect\")\n            ok = False\n        if should_return_json:\n            return Response.json(\n                {\n                    \"ok\": ok,\n                    \"message\": message,\n                    \"redirect\": redirect_url,\n                }\n            )\n        else:\n            datasette.add_message(request, message, message_type)\n            return Response.redirect(redirect_url or request.path)\n\n    async def get(self, request, datasette):\n        from datasette.app import TableNotFound\n\n        await datasette.refresh_schemas()\n\n        db = await datasette.resolve_database(request)\n        database = db.name\n\n        # Get all tables/views this actor can see in bulk with private flag\n        allowed_tables_page = await datasette.allowed_resources(\n            \"view-table\",\n            request.actor,\n            parent=database,\n            include_is_private=True,\n            limit=1000,\n        )\n        # Create lookup dict for quick access\n        allowed_dict = {r.child: r for r in allowed_tables_page.resources}\n\n        # Are we a canned query?\n        canned_query = None\n        canned_query_write = False\n        if \"table\" in request.url_vars:\n            try:\n                await datasette.resolve_table(request)\n            except TableNotFound as table_not_found:\n                # Was this actually a canned query?\n                canned_query = await datasette.get_canned_query(\n                    table_not_found.database_name, table_not_found.table, request.actor\n                )\n                if canned_query is None:\n                    raise\n                canned_query_write = bool(canned_query.get(\"write\"))\n\n        private = False\n        if canned_query:\n            # Respect canned query permissions\n            visible, private = await datasette.check_visibility(\n                request.actor,\n                action=\"view-query\",\n                resource=QueryResource(database=database, query=canned_query[\"name\"]),\n            )\n            if not visible:\n                raise Forbidden(\"You do not have permission to view this query\")\n\n        else:\n            await datasette.ensure_permission(\n                action=\"execute-sql\",\n                resource=DatabaseResource(database=database),\n                actor=request.actor,\n            )\n\n        # Flattened because of ?sql=&name1=value1&name2=value2 feature\n        params = {key: request.args.get(key) for key in request.args}\n        sql = None\n\n        if canned_query:\n            sql = canned_query[\"sql\"]\n        elif \"sql\" in params:\n            sql = params.pop(\"sql\")\n\n        # Extract any :named parameters\n        named_parameters = []\n        if canned_query and canned_query.get(\"params\"):\n            named_parameters = canned_query[\"params\"]\n        if not named_parameters:\n            named_parameters = derive_named_parameters(sql)\n        named_parameter_values = {\n            named_parameter: params.get(named_parameter) or \"\"\n            for named_parameter in named_parameters\n            if not named_parameter.startswith(\"_\")\n        }\n        # Set to blank string if missing from params\n        for named_parameter in named_parameters:\n            if named_parameter not in params and not named_parameter.startswith(\"_\"):\n                params[named_parameter] = \"\"\n\n        extra_args = {}\n        if params.get(\"_timelimit\"):\n            extra_args[\"custom_time_limit\"] = int(params[\"_timelimit\"])\n\n        format_ = request.url_vars.get(\"format\") or \"html\"\n\n        query_error = None\n        results = None\n        rows = []\n        columns = []\n\n        params_for_query = params\n\n        if not canned_query_write:\n            try:\n                if not canned_query:\n                    # For regular queries we only allow SELECT, plus other rules\n                    validate_sql_select(sql)\n                else:\n                    # Canned queries can run magic parameters\n                    params_for_query = MagicParameters(sql, params, request, datasette)\n                    await params_for_query.execute_params()\n                results = await datasette.execute(\n                    database, sql, params_for_query, truncate=True, **extra_args\n                )\n                columns = results.columns\n                rows = results.rows\n            except QueryInterrupted as ex:\n                raise DatasetteError(\n                    textwrap.dedent(\"\"\"\n                    
    SQL query took too long. The time limit is controlled by the\n                    sql_time_limit_ms\n                    configuration option.
    \n                    \n                    \n                \"\"\".format(markupsafe.escape(ex.sql))).strip(),\n                    title=\"SQL Interrupted\",\n                    status=400,\n                    message_is_html=True,\n                )\n            except sqlite3.DatabaseError as ex:\n                query_error = str(ex)\n                results = None\n                rows = []\n                columns = []\n            except (sqlite3.OperationalError, InvalidSql) as ex:\n                raise DatasetteError(str(ex), title=\"Invalid SQL\", status=400)\n            except sqlite3.OperationalError as ex:\n                raise DatasetteError(str(ex))\n            except DatasetteError:\n                raise\n\n        # Handle formats from plugins\n        if format_ == \"csv\":\n\n            async def fetch_data_for_csv(request, _next=None):\n                results = await db.execute(sql, params, truncate=True)\n                data = {\"rows\": results.rows, \"columns\": results.columns}\n                return data, None, None\n\n            return await stream_csv(datasette, fetch_data_for_csv, request, db.name)\n        elif format_ in datasette.renderers.keys():\n            # Dispatch request to the correct output format renderer\n            # (CSV is not handled here due to streaming)\n            result = call_with_supported_arguments(\n                datasette.renderers[format_][0],\n                datasette=datasette,\n                columns=columns,\n                rows=rows,\n                sql=sql,\n                query_name=canned_query[\"name\"] if canned_query else None,\n                database=database,\n                table=None,\n                request=request,\n                view_name=\"table\",\n                truncated=results.truncated if results else False,\n                error=query_error,\n                # These will be deprecated in Datasette 1.0:\n                args=request.args,\n                data={\"ok\": True, \"rows\": rows, \"columns\": columns},\n            )\n            if asyncio.iscoroutine(result):\n                result = await result\n            if result is None:\n                raise NotFound(\"No data\")\n            if isinstance(result, dict):\n                r = Response(\n                    body=result.get(\"body\"),\n                    status=result.get(\"status_code\") or 200,\n                    content_type=result.get(\"content_type\", \"text/plain\"),\n                    headers=result.get(\"headers\"),\n                )\n            elif isinstance(result, Response):\n                r = result\n                # if status_code is not None:\n                #     # Over-ride the status code\n                #     r.status = status_code\n            else:\n                assert False, f\"{result} should be dict or Response\"\n        elif format_ == \"html\":\n            headers = {}\n            templates = [f\"query-{to_css_class(database)}.html\", \"query.html\"]\n            if canned_query:\n                templates.insert(\n                    0,\n                    f\"query-{to_css_class(database)}-{to_css_class(canned_query['name'])}.html\",\n                )\n\n            environment = datasette.get_jinja_environment(request)\n            template = environment.select_template(templates)\n            alternate_url_json = datasette.absolute_url(\n                request,\n                datasette.urls.path(path_with_format(request=request, format=\"json\")),\n            )\n            data = {}\n            headers.update(\n                {\n                    \"Link\": '<{}>; rel=\"alternate\"; type=\"application/json+datasette\"'.format(\n                        alternate_url_json\n                    )\n                }\n            )\n            metadata = await datasette.get_database_metadata(database)\n\n            renderers = {}\n            for key, (_, can_render) in datasette.renderers.items():\n                it_can_render = call_with_supported_arguments(\n                    can_render,\n                    datasette=datasette,\n                    columns=data.get(\"columns\") or [],\n                    rows=data.get(\"rows\") or [],\n                    sql=data.get(\"query\", {}).get(\"sql\", None),\n                    query_name=data.get(\"query_name\"),\n                    database=database,\n                    table=data.get(\"table\"),\n                    request=request,\n                    view_name=\"database\",\n                )\n                it_can_render = await await_me_maybe(it_can_render)\n                if it_can_render:\n                    renderers[key] = datasette.urls.path(\n                        path_with_format(request=request, format=key)\n                    )\n\n            allow_execute_sql = await datasette.allowed(\n                action=\"execute-sql\",\n                resource=DatabaseResource(database=database),\n                actor=request.actor,\n            )\n\n            show_hide_hidden = \"\"\n            if canned_query and canned_query.get(\"hide_sql\"):\n                if bool(params.get(\"_show_sql\")):\n                    show_hide_link = path_with_removed_args(request, {\"_show_sql\"})\n                    show_hide_text = \"hide\"\n                    show_hide_hidden = (\n                        ''\n                    )\n                else:\n                    show_hide_link = path_with_added_args(request, {\"_show_sql\": 1})\n                    show_hide_text = \"show\"\n            else:\n                if bool(params.get(\"_hide_sql\")):\n                    show_hide_link = path_with_removed_args(request, {\"_hide_sql\"})\n                    show_hide_text = \"show\"\n                    show_hide_hidden = (\n                        ''\n                    )\n                else:\n                    show_hide_link = path_with_added_args(request, {\"_hide_sql\": 1})\n                    show_hide_text = \"hide\"\n            hide_sql = show_hide_text == \"show\"\n\n            # Show 'Edit SQL' button only if:\n            # - User is allowed to execute SQL\n            # - SQL is an approved SELECT statement\n            # - No magic parameters, so no :_ in the SQL string\n            edit_sql_url = None\n            is_validated_sql = False\n            try:\n                validate_sql_select(sql)\n                is_validated_sql = True\n            except InvalidSql:\n                pass\n            if allow_execute_sql and is_validated_sql and \":_\" not in sql:\n                edit_sql_url = (\n                    datasette.urls.database(database)\n                    + \"/-/query\"\n                    + \"?\"\n                    + urlencode(\n                        {\n                            **{\n                                \"sql\": sql,\n                            },\n                            **named_parameter_values,\n                        }\n                    )\n                )\n\n            async def query_actions():\n                query_actions = []\n                for hook in pm.hook.query_actions(\n                    datasette=datasette,\n                    actor=request.actor,\n                    database=database,\n                    query_name=canned_query[\"name\"] if canned_query else None,\n                    request=request,\n                    sql=sql,\n                    params=params,\n                ):\n                    extra_links = await await_me_maybe(hook)\n                    if extra_links:\n                        query_actions.extend(extra_links)\n                return query_actions\n\n            r = Response.html(\n                await datasette.render_template(\n                    template,\n                    QueryContext(\n                        database=database,\n                        database_color=db.color,\n                        query={\n                            \"sql\": sql,\n                            \"params\": params,\n                        },\n                        canned_query=canned_query[\"name\"] if canned_query else None,\n                        private=private,\n                        canned_query_write=canned_query_write,\n                        db_is_immutable=not db.is_mutable,\n                        error=query_error,\n                        hide_sql=hide_sql,\n                        show_hide_link=datasette.urls.path(show_hide_link),\n                        show_hide_text=show_hide_text,\n                        editable=not canned_query,\n                        allow_execute_sql=allow_execute_sql,\n                        tables=await get_tables(datasette, request, db, allowed_dict),\n                        named_parameter_values=named_parameter_values,\n                        edit_sql_url=edit_sql_url,\n                        display_rows=await display_rows(\n                            datasette, database, request, rows, columns\n                        ),\n                        table_columns=(\n                            await _table_columns(datasette, database)\n                            if allow_execute_sql\n                            else {}\n                        ),\n                        columns=columns,\n                        renderers=renderers,\n                        url_csv=datasette.urls.path(\n                            path_with_format(\n                                request=request, format=\"csv\", extra_qs={\"_size\": \"max\"}\n                            )\n                        ),\n                        show_hide_hidden=markupsafe.Markup(show_hide_hidden),\n                        metadata=canned_query or metadata,\n                        alternate_url_json=alternate_url_json,\n                        select_templates=[\n                            f\"{'*' if template_name == template.name else ''}{template_name}\"\n                            for template_name in templates\n                        ],\n                        top_query=make_slot_function(\n                            \"top_query\", datasette, request, database=database, sql=sql\n                        ),\n                        top_canned_query=make_slot_function(\n                            \"top_canned_query\",\n                            datasette,\n                            request,\n                            database=database,\n                            query_name=canned_query[\"name\"] if canned_query else None,\n                        ),\n                        query_actions=query_actions,\n                    ),\n                    request=request,\n                    view_name=\"database\",\n                ),\n                headers=headers,\n            )\n        else:\n            assert False, \"Invalid format: {}\".format(format_)\n        if datasette.cors:\n            add_cors_headers(r.headers)\n        return r\n\n\nclass MagicParameters(dict):\n    def __init__(self, sql, data, request, datasette):\n        super().__init__(data)\n        self._sql = sql\n        self._request = request\n        self._magics = dict(\n            itertools.chain.from_iterable(\n                pm.hook.register_magic_parameters(datasette=datasette)\n            )\n        )\n        self._prepared = {}\n\n    async def execute_params(self):\n        for key in derive_named_parameters(self._sql):\n            if key.startswith(\"_\") and key.count(\"_\") >= 2:\n                prefix, suffix = key[1:].split(\"_\", 1)\n                if prefix in self._magics:\n                    result = await await_me_maybe(\n                        self._magics[prefix](suffix, self._request)\n                    )\n                    self._prepared[key] = result\n\n    def __len__(self):\n        # Workaround for 'Incorrect number of bindings' error\n        # https://github.com/simonw/datasette/issues/967#issuecomment-692951144\n        return super().__len__() or 1\n\n    def __getitem__(self, key):\n        if key.startswith(\"_\") and key.count(\"_\") >= 2:\n            if key in self._prepared:\n                return self._prepared[key]\n            # Try the other route\n            prefix, suffix = key[1:].split(\"_\", 1)\n            if prefix in self._magics:\n                try:\n                    return self._magics[prefix](suffix, self._request)\n                except KeyError:\n                    return super().__getitem__(key)\n        else:\n            return super().__getitem__(key)\n\n\nclass TableCreateView(BaseView):\n    name = \"table-create\"\n\n    _valid_keys = {\n        \"table\",\n        \"rows\",\n        \"row\",\n        \"columns\",\n        \"pk\",\n        \"pks\",\n        \"ignore\",\n        \"replace\",\n        \"alter\",\n    }\n    _supported_column_types = {\n        \"text\",\n        \"integer\",\n        \"float\",\n        \"blob\",\n    }\n    # Any string that does not contain a newline or start with sqlite_\n    _table_name_re = re.compile(r\"^(?!sqlite_)[^\\n]+$\")\n\n    def __init__(self, datasette):\n        self.ds = datasette\n\n    async def post(self, request):\n        db = await self.ds.resolve_database(request)\n        database_name = db.name\n\n        # Must have create-table permission\n        if not await self.ds.allowed(\n            action=\"create-table\",\n            resource=DatabaseResource(database=database_name),\n            actor=request.actor,\n        ):\n            return _error([\"Permission denied\"], 403)\n\n        body = await request.post_body()\n        try:\n            data = json.loads(body)\n        except json.JSONDecodeError as e:\n            return _error([\"Invalid JSON: {}\".format(e)])\n\n        if not isinstance(data, dict):\n            return _error([\"JSON must be an object\"])\n\n        invalid_keys = set(data.keys()) - self._valid_keys\n        if invalid_keys:\n            return _error([\"Invalid keys: {}\".format(\", \".join(invalid_keys))])\n\n        # ignore and replace are mutually exclusive\n        if data.get(\"ignore\") and data.get(\"replace\"):\n            return _error([\"ignore and replace are mutually exclusive\"])\n\n        # ignore and replace only allowed with row or rows\n        if \"ignore\" in data or \"replace\" in data:\n            if not data.get(\"row\") and not data.get(\"rows\"):\n                return _error([\"ignore and replace require row or rows\"])\n\n        # ignore and replace require pk or pks\n        if \"ignore\" in data or \"replace\" in data:\n            if not data.get(\"pk\") and not data.get(\"pks\"):\n                return _error([\"ignore and replace require pk or pks\"])\n\n        ignore = data.get(\"ignore\")\n        replace = data.get(\"replace\")\n\n        if replace:\n            # Must have update-row permission\n            if not await self.ds.allowed(\n                action=\"update-row\",\n                resource=DatabaseResource(database=database_name),\n                actor=request.actor,\n            ):\n                return _error([\"Permission denied: need update-row\"], 403)\n\n        table_name = data.get(\"table\")\n        if not table_name:\n            return _error([\"Table is required\"])\n\n        if not self._table_name_re.match(table_name):\n            return _error([\"Invalid table name\"])\n\n        table_exists = await db.table_exists(data[\"table\"])\n        columns = data.get(\"columns\")\n        rows = data.get(\"rows\")\n        row = data.get(\"row\")\n        if not columns and not rows and not row:\n            return _error([\"columns, rows or row is required\"])\n\n        if rows and row:\n            return _error([\"Cannot specify both rows and row\"])\n\n        if rows or row:\n            # Must have insert-row permission\n            if not await self.ds.allowed(\n                action=\"insert-row\",\n                resource=DatabaseResource(database=database_name),\n                actor=request.actor,\n            ):\n                return _error([\"Permission denied: need insert-row\"], 403)\n\n        alter = False\n        if rows or row:\n            if not table_exists:\n                # if table is being created for the first time, alter=True\n                alter = True\n            else:\n                # alter=True only if they request it AND they have permission\n                if data.get(\"alter\"):\n                    if not await self.ds.allowed(\n                        action=\"alter-table\",\n                        resource=DatabaseResource(database=database_name),\n                        actor=request.actor,\n                    ):\n                        return _error([\"Permission denied: need alter-table\"], 403)\n                    alter = True\n\n        if columns:\n            if rows or row:\n                return _error([\"Cannot specify columns with rows or row\"])\n            if not isinstance(columns, list):\n                return _error([\"columns must be a list\"])\n            for column in columns:\n                if not isinstance(column, dict):\n                    return _error([\"columns must be a list of objects\"])\n                if not column.get(\"name\") or not isinstance(column.get(\"name\"), str):\n                    return _error([\"Column name is required\"])\n                if not column.get(\"type\"):\n                    column[\"type\"] = \"text\"\n                if column[\"type\"] not in self._supported_column_types:\n                    return _error(\n                        [\"Unsupported column type: {}\".format(column[\"type\"])]\n                    )\n            # No duplicate column names\n            dupes = {c[\"name\"] for c in columns if columns.count(c) > 1}\n            if dupes:\n                return _error([\"Duplicate column name: {}\".format(\", \".join(dupes))])\n\n        if row:\n            rows = [row]\n\n        if rows:\n            if not isinstance(rows, list):\n                return _error([\"rows must be a list\"])\n            for row in rows:\n                if not isinstance(row, dict):\n                    return _error([\"rows must be a list of objects\"])\n\n        pk = data.get(\"pk\")\n        pks = data.get(\"pks\")\n\n        if pk and pks:\n            return _error([\"Cannot specify both pk and pks\"])\n        if pk:\n            if not isinstance(pk, str):\n                return _error([\"pk must be a string\"])\n        if pks:\n            if not isinstance(pks, list):\n                return _error([\"pks must be a list\"])\n            for pk in pks:\n                if not isinstance(pk, str):\n                    return _error([\"pks must be a list of strings\"])\n\n        # If table exists already, read pks from that instead\n        if table_exists:\n            actual_pks = await db.primary_keys(table_name)\n            # if pk passed and table already exists check it does not change\n            bad_pks = False\n            if len(actual_pks) == 1 and data.get(\"pk\") and data[\"pk\"] != actual_pks[0]:\n                bad_pks = True\n            elif (\n                len(actual_pks) > 1\n                and data.get(\"pks\")\n                and set(data[\"pks\"]) != set(actual_pks)\n            ):\n                bad_pks = True\n            if bad_pks:\n                return _error([\"pk cannot be changed for existing table\"])\n            pks = actual_pks\n\n        initial_schema = None\n        if table_exists:\n            initial_schema = await db.execute_fn(\n                lambda conn: sqlite_utils.Database(conn)[table_name].schema\n            )\n\n        def create_table(conn):\n            table = sqlite_utils.Database(conn)[table_name]\n            if rows:\n                table.insert_all(\n                    rows, pk=pks or pk, ignore=ignore, replace=replace, alter=alter\n                )\n            else:\n                table.create(\n                    {c[\"name\"]: c[\"type\"] for c in columns},\n                    pk=pks or pk,\n                )\n            return table.schema\n\n        try:\n            schema = await db.execute_write_fn(create_table, request=request)\n        except Exception as e:\n            return _error([str(e)])\n\n        if initial_schema is not None and initial_schema != schema:\n            await self.ds.track_event(\n                AlterTableEvent(\n                    request.actor,\n                    database=database_name,\n                    table=table_name,\n                    before_schema=initial_schema,\n                    after_schema=schema,\n                )\n            )\n\n        table_url = self.ds.absolute_url(\n            request, self.ds.urls.table(db.name, table_name)\n        )\n        table_api_url = self.ds.absolute_url(\n            request, self.ds.urls.table(db.name, table_name, format=\"json\")\n        )\n        details = {\n            \"ok\": True,\n            \"database\": db.name,\n            \"table\": table_name,\n            \"table_url\": table_url,\n            \"table_api_url\": table_api_url,\n            \"schema\": schema,\n        }\n        if rows:\n            details[\"row_count\"] = len(rows)\n\n        if not table_exists:\n            # Only log creation if we created a table\n            await self.ds.track_event(\n                CreateTableEvent(\n                    request.actor, database=db.name, table=table_name, schema=schema\n                )\n            )\n        if rows:\n            await self.ds.track_event(\n                InsertRowsEvent(\n                    request.actor,\n                    database=db.name,\n                    table=table_name,\n                    num_rows=len(rows),\n                    ignore=ignore,\n                    replace=replace,\n                )\n            )\n        return Response.json(details, status=201)\n\n\nasync def _table_columns(datasette, database_name):\n    internal_db = datasette.get_internal_database()\n    result = await internal_db.execute(\n        \"select table_name, name from catalog_columns where database_name = ?\",\n        [database_name],\n    )\n    table_columns = {}\n    for row in result.rows:\n        table_columns.setdefault(row[\"table_name\"], []).append(row[\"name\"])\n    # Add views\n    db = datasette.get_database(database_name)\n    for view_name in await db.view_names():\n        table_columns[view_name] = []\n    return table_columns\n\n\nasync def display_rows(datasette, database, request, rows, columns):\n    display_rows = []\n    truncate_cells = datasette.setting(\"truncate_cells_html\")\n    for row in rows:\n        display_row = []\n        for column, value in zip(columns, row):\n            display_value = value\n            # Let the plugins have a go\n            # pylint: disable=no-member\n            plugin_display_value = None\n            for candidate in pm.hook.render_cell(\n                row=row,\n                value=value,\n                column=column,\n                table=None,\n                pks=[],\n                database=database,\n                datasette=datasette,\n                request=request,\n                column_type=None,\n            ):\n                candidate = await await_me_maybe(candidate)\n                if candidate is not None:\n                    plugin_display_value = candidate\n                    break\n            if plugin_display_value is not None:\n                display_value = plugin_display_value\n            else:\n                if value in (\"\", None):\n                    display_value = markupsafe.Markup(\" \")\n                elif is_url(str(display_value).strip()):\n                    display_value = markupsafe.Markup(\n                        '{truncated_url}'.format(\n                            url=markupsafe.escape(value.strip()),\n                            truncated_url=markupsafe.escape(\n                                truncate_url(value.strip(), truncate_cells)\n                            ),\n                        )\n                    )\n                elif isinstance(display_value, bytes):\n                    blob_url = path_with_format(\n                        request=request,\n                        format=\"blob\",\n                        extra_qs={\n                            \"_blob_column\": column,\n                            \"_blob_hash\": hashlib.sha256(display_value).hexdigest(),\n                        },\n                    )\n                    formatted = format_bytes(len(value))\n                    display_value = markupsafe.Markup(\n                        '<Binary: {:,} byte{}>'.format(\n                            blob_url,\n                            (\n                                ' title=\"{}\"'.format(formatted)\n                                if \"bytes\" not in formatted\n                                else \"\"\n                            ),\n                            len(value),\n                            \"\" if len(value) == 1 else \"s\",\n                        )\n                    )\n                else:\n                    display_value = str(value)\n                    if truncate_cells and len(display_value) > truncate_cells:\n                        display_value = display_value[:truncate_cells] + \"\\u2026\"\n            display_row.append(display_value)\n        display_rows.append(display_row)\n    return display_rows\n"
  },
  {
    "path": "datasette/views/index.py",
    "content": "import json\n\nfrom datasette.plugins import pm\nfrom datasette.utils import (\n    add_cors_headers,\n    await_me_maybe,\n    make_slot_function,\n    CustomJSONEncoder,\n)\nfrom datasette.utils.asgi import Response\nfrom datasette.version import __version__\n\nfrom .base import BaseView\n\n# Truncate table list on homepage at:\nTRUNCATE_AT = 5\n\n# Only attempt counts if database less than this size in bytes:\nCOUNT_DB_SIZE_LIMIT = 100 * 1024 * 1024\n\n\nclass IndexView(BaseView):\n    name = \"index\"\n\n    async def get(self, request):\n        as_format = request.url_vars[\"format\"]\n        await self.ds.ensure_permission(action=\"view-instance\", actor=request.actor)\n\n        # Get all allowed databases and tables in bulk\n        db_page = await self.ds.allowed_resources(\n            \"view-database\", request.actor, include_is_private=True\n        )\n        allowed_databases = [r async for r in db_page.all()]\n        allowed_db_dict = {r.parent: r for r in allowed_databases}\n\n        # Group tables by database\n        tables_by_db = {}\n        table_page = await self.ds.allowed_resources(\n            \"view-table\", request.actor, include_is_private=True\n        )\n        async for t in table_page.all():\n            if t.parent not in tables_by_db:\n                tables_by_db[t.parent] = {}\n            tables_by_db[t.parent][t.child] = t\n\n        databases = []\n        # Iterate over allowed databases instead of all databases\n        for name in allowed_db_dict.keys():\n            db = self.ds.databases[name]\n            database_private = allowed_db_dict[name].private\n\n            # Get allowed tables/views for this database\n            allowed_for_db = tables_by_db.get(name, {})\n\n            # Get table names from allowed set instead of db.table_names()\n            table_names = [child_name for child_name in allowed_for_db.keys()]\n\n            hidden_table_names = set(await db.hidden_table_names())\n\n            # Determine which allowed items are views\n            view_names_set = set(await db.view_names())\n            views = [\n                {\"name\": child_name, \"private\": resource.private}\n                for child_name, resource in allowed_for_db.items()\n                if child_name in view_names_set\n            ]\n\n            # Filter to just tables (not views) for table processing\n            table_names = [name for name in table_names if name not in view_names_set]\n\n            # Perform counts only for immutable or DBS with <= COUNT_TABLE_LIMIT tables\n            table_counts = {}\n            if not db.is_mutable or db.size < COUNT_DB_SIZE_LIMIT:\n                table_counts = await db.table_counts(10)\n                # If any of these are None it means at least one timed out - ignore them all\n                if any(v is None for v in table_counts.values()):\n                    table_counts = {}\n\n            tables = {}\n            for table in table_names:\n                # Check if table is in allowed set\n                if table not in allowed_for_db:\n                    continue\n\n                table_columns = await db.table_columns(table)\n                tables[table] = {\n                    \"name\": table,\n                    \"columns\": table_columns,\n                    \"primary_keys\": await db.primary_keys(table),\n                    \"count\": table_counts.get(table),\n                    \"hidden\": table in hidden_table_names,\n                    \"fts_table\": await db.fts_table(table),\n                    \"num_relationships_for_sorting\": 0,\n                    \"private\": allowed_for_db[table].private,\n                }\n\n            if request.args.get(\"_sort\") == \"relationships\" or not table_counts:\n                # We will be sorting by number of relationships, so populate that field\n                all_foreign_keys = await db.get_all_foreign_keys()\n                for table, foreign_keys in all_foreign_keys.items():\n                    if table in tables.keys():\n                        count = len(foreign_keys[\"incoming\"] + foreign_keys[\"outgoing\"])\n                        tables[table][\"num_relationships_for_sorting\"] = count\n\n            hidden_tables = [t for t in tables.values() if t[\"hidden\"]]\n            visible_tables = [t for t in tables.values() if not t[\"hidden\"]]\n\n            tables_and_views_truncated = list(\n                sorted(\n                    (t for t in tables.values() if t not in hidden_tables),\n                    key=lambda t: (\n                        t[\"num_relationships_for_sorting\"],\n                        t[\"count\"] or 0,\n                        t[\"name\"],\n                    ),\n                    reverse=True,\n                )[:TRUNCATE_AT]\n            )\n\n            # Only add views if this is less than TRUNCATE_AT\n            if len(tables_and_views_truncated) < TRUNCATE_AT:\n                num_views_to_add = TRUNCATE_AT - len(tables_and_views_truncated)\n                for view in views[:num_views_to_add]:\n                    tables_and_views_truncated.append(view)\n\n            databases.append(\n                {\n                    \"name\": name,\n                    \"hash\": db.hash,\n                    \"color\": db.color,\n                    \"path\": self.ds.urls.database(name),\n                    \"tables_and_views_truncated\": tables_and_views_truncated,\n                    \"tables_and_views_more\": (len(visible_tables) + len(views))\n                    > TRUNCATE_AT,\n                    \"tables_count\": len(visible_tables),\n                    \"table_rows_sum\": sum((t[\"count\"] or 0) for t in visible_tables),\n                    \"show_table_row_counts\": bool(table_counts),\n                    \"hidden_table_rows_sum\": sum(\n                        t[\"count\"] for t in hidden_tables if t[\"count\"] is not None\n                    ),\n                    \"hidden_tables_count\": len(hidden_tables),\n                    \"views_count\": len(views),\n                    \"private\": database_private,\n                }\n            )\n\n        if as_format:\n            headers = {}\n            if self.ds.cors:\n                add_cors_headers(headers)\n            return Response(\n                json.dumps(\n                    {\n                        \"databases\": {db[\"name\"]: db for db in databases},\n                        \"metadata\": await self.ds.get_instance_metadata(),\n                    },\n                    cls=CustomJSONEncoder,\n                ),\n                content_type=\"application/json; charset=utf-8\",\n                headers=headers,\n            )\n        else:\n            homepage_actions = []\n            for hook in pm.hook.homepage_actions(\n                datasette=self.ds,\n                actor=request.actor,\n                request=request,\n            ):\n                extra_links = await await_me_maybe(hook)\n                if extra_links:\n                    homepage_actions.extend(extra_links)\n            alternative_homepage = request.path == \"/-/\"\n            return await self.render(\n                [\"default:index.html\" if alternative_homepage else \"index.html\"],\n                request=request,\n                context={\n                    \"databases\": databases,\n                    \"metadata\": await self.ds.get_instance_metadata(),\n                    \"datasette_version\": __version__,\n                    \"private\": not await self.ds.allowed(\n                        action=\"view-instance\", actor=None\n                    ),\n                    \"top_homepage\": make_slot_function(\n                        \"top_homepage\", self.ds, request\n                    ),\n                    \"homepage_actions\": homepage_actions,\n                    \"noindex\": request.path == \"/-/\",\n                },\n            )\n"
  },
  {
    "path": "datasette/views/row.py",
    "content": "from datasette.utils.asgi import NotFound, Forbidden, Response\nfrom datasette.database import QueryInterrupted\nfrom datasette.events import UpdateRowEvent, DeleteRowEvent\nfrom datasette.resources import TableResource\nfrom .base import DataView, BaseView, _error\nfrom datasette.utils import (\n    await_me_maybe,\n    CustomRow,\n    make_slot_function,\n    to_css_class,\n    escape_sqlite,\n)\nfrom datasette.plugins import pm\nimport json\nimport markupsafe\nimport sqlite_utils\nfrom .table import display_columns_and_rows, _get_extras\n\n\nclass RowView(DataView):\n    name = \"row\"\n\n    async def data(self, request, default_labels=False):\n        resolved = await self.ds.resolve_row(request)\n        db = resolved.db\n        database = db.name\n        table = resolved.table\n        pk_values = resolved.pk_values\n\n        # Ensure user has permission to view this row\n        visible, private = await self.ds.check_visibility(\n            request.actor,\n            action=\"view-table\",\n            resource=TableResource(database=database, table=table),\n        )\n        if not visible:\n            raise Forbidden(\"You do not have permission to view this table\")\n\n        results = await resolved.db.execute(\n            resolved.sql, resolved.params, truncate=True\n        )\n        columns = [r[0] for r in results.description]\n        rows = list(results.rows)\n        if not rows:\n            raise NotFound(f\"Record not found: {pk_values}\")\n\n        pks = resolved.pks\n\n        async def template_data():\n            # Reorder columns so primary keys come first\n            pk_set = set(pks)\n            pk_cols = [d for d in results.description if d[0] in pk_set]\n            non_pk_cols = [d for d in results.description if d[0] not in pk_set]\n            reordered_description = pk_cols + non_pk_cols\n            reordered_columns = [d[0] for d in reordered_description]\n\n            # Reorder row data to match\n            reordered_rows = []\n            for row in rows:\n                new_row = CustomRow(reordered_columns)\n                for col in reordered_columns:\n                    new_row[col] = row[col]\n                reordered_rows.append(new_row)\n\n            # Expand foreign key columns into dicts so display_columns_and_rows\n            # renders them as hyperlinks, matching the table view behavior\n            expanded_rows = reordered_rows\n            for fk in await db.foreign_keys_for_table(table):\n                column = fk[\"column\"]\n                if column not in reordered_columns:\n                    continue\n                column_index = reordered_columns.index(column)\n                values = [row[column_index] for row in expanded_rows]\n                expanded_labels = await self.ds.expand_foreign_keys(\n                    request.actor, database, table, column, values\n                )\n                if expanded_labels:\n                    new_rows = []\n                    for row in expanded_rows:\n                        new_row = CustomRow(reordered_columns)\n                        for col in reordered_columns:\n                            value = row[col]\n                            if (\n                                col == column\n                                and (col, value) in expanded_labels\n                                and value is not None\n                            ):\n                                new_row[col] = {\n                                    \"value\": value,\n                                    \"label\": expanded_labels[(col, value)],\n                                }\n                            else:\n                                new_row[col] = value\n                        new_rows.append(new_row)\n                    expanded_rows = new_rows\n\n            display_columns, display_rows = await display_columns_and_rows(\n                self.ds,\n                database,\n                table,\n                reordered_description,\n                expanded_rows,\n                link_column=False,\n                truncate_cells=0,\n                request=request,\n            )\n            for column in display_columns:\n                column[\"sortable\"] = False\n\n            # Bold primary key cell values\n            for row in display_rows:\n                for cell in row:\n                    if cell[\"column\"] in pk_set:\n                        cell[\"value\"] = markupsafe.Markup(\n                            \"{}\".format(cell[\"value\"])\n                        )\n\n            row_actions = []\n            for hook in pm.hook.row_actions(\n                datasette=self.ds,\n                actor=request.actor,\n                request=request,\n                database=database,\n                table=table,\n                row=rows[0],\n            ):\n                extra_links = await await_me_maybe(hook)\n                if extra_links:\n                    row_actions.extend(extra_links)\n\n            return {\n                \"private\": private,\n                \"columns\": reordered_columns,\n                \"foreign_key_tables\": await self.foreign_key_tables(\n                    database, table, pk_values\n                ),\n                \"database_color\": db.color,\n                \"display_columns\": display_columns,\n                \"display_rows\": display_rows,\n                \"custom_table_templates\": [\n                    f\"_table-{to_css_class(database)}-{to_css_class(table)}.html\",\n                    f\"_table-row-{to_css_class(database)}-{to_css_class(table)}.html\",\n                    \"_table.html\",\n                ],\n                \"row_actions\": row_actions,\n                \"top_row\": make_slot_function(\n                    \"top_row\",\n                    self.ds,\n                    request,\n                    database=resolved.db.name,\n                    table=resolved.table,\n                    row=rows[0],\n                ),\n                \"metadata\": {},\n            }\n\n        data = {\n            \"ok\": True,\n            \"database\": database,\n            \"table\": table,\n            \"rows\": rows,\n            \"columns\": columns,\n            \"primary_keys\": resolved.pks,\n            \"primary_key_values\": pk_values,\n        }\n\n        # Handle _extra parameter (new style)\n        extras = _get_extras(request)\n\n        # Also support legacy _extras parameter for backward compatibility\n        if \"foreign_key_tables\" in (request.args.get(\"_extras\") or \"\").split(\",\"):\n            extras.add(\"foreign_key_tables\")\n\n        # Process extras\n        if \"foreign_key_tables\" in extras:\n            data[\"foreign_key_tables\"] = await self.foreign_key_tables(\n                database, table, pk_values\n            )\n\n        if \"render_cell\" in extras:\n            # Call render_cell plugin hook for each cell\n            ct_map = await self.ds.get_column_types(database, table)\n            rendered_rows = []\n            for row in rows:\n                rendered_row = {}\n                for value, column in zip(row, columns):\n                    ct = ct_map.get(column)\n                    plugin_display_value = None\n                    # Try column type render_cell first\n                    if ct:\n                        candidate = await ct.render_cell(\n                            value=value,\n                            column=column,\n                            table=table,\n                            database=database,\n                            datasette=self.ds,\n                            request=request,\n                        )\n                        if candidate is not None:\n                            plugin_display_value = candidate\n                    if plugin_display_value is None:\n                        for candidate in pm.hook.render_cell(\n                            row=row,\n                            value=value,\n                            column=column,\n                            table=table,\n                            pks=resolved.pks,\n                            database=database,\n                            datasette=self.ds,\n                            request=request,\n                            column_type=ct,\n                        ):\n                            candidate = await await_me_maybe(candidate)\n                            if candidate is not None:\n                                plugin_display_value = candidate\n                                break\n                    if plugin_display_value:\n                        rendered_row[column] = str(plugin_display_value)\n                rendered_rows.append(rendered_row)\n            data[\"render_cell\"] = rendered_rows\n\n        return (\n            data,\n            template_data,\n            (\n                f\"row-{to_css_class(database)}-{to_css_class(table)}.html\",\n                \"row.html\",\n            ),\n        )\n\n    async def foreign_key_tables(self, database, table, pk_values):\n        if len(pk_values) != 1:\n            return []\n        db = self.ds.databases[database]\n        all_foreign_keys = await db.get_all_foreign_keys()\n        foreign_keys = all_foreign_keys[table][\"incoming\"]\n        if len(foreign_keys) == 0:\n            return []\n\n        sql = \"select \" + \", \".join(\n            [\n                \"(select count(*) from {table} where {column}=:id)\".format(\n                    table=escape_sqlite(fk[\"other_table\"]),\n                    column=escape_sqlite(fk[\"other_column\"]),\n                )\n                for fk in foreign_keys\n            ]\n        )\n        try:\n            rows = list(await db.execute(sql, {\"id\": pk_values[0]}))\n        except QueryInterrupted:\n            # Almost certainly hit the timeout\n            return []\n\n        foreign_table_counts = dict(\n            zip(\n                [(fk[\"other_table\"], fk[\"other_column\"]) for fk in foreign_keys],\n                list(rows[0]),\n            )\n        )\n        foreign_key_tables = []\n        for fk in foreign_keys:\n            count = (\n                foreign_table_counts.get((fk[\"other_table\"], fk[\"other_column\"])) or 0\n            )\n            key = fk[\"other_column\"]\n            if key.startswith(\"_\"):\n                key += \"__exact\"\n            link = \"{}?{}={}\".format(\n                self.ds.urls.table(database, fk[\"other_table\"]),\n                key,\n                \",\".join(pk_values),\n            )\n            foreign_key_tables.append({**fk, **{\"count\": count, \"link\": link}})\n        return foreign_key_tables\n\n\nclass RowError(Exception):\n    def __init__(self, error):\n        self.error = error\n\n\nasync def _resolve_row_and_check_permission(datasette, request, permission):\n    from datasette.app import DatabaseNotFound, TableNotFound, RowNotFound\n\n    try:\n        resolved = await datasette.resolve_row(request)\n    except DatabaseNotFound as e:\n        return False, _error([\"Database not found: {}\".format(e.database_name)], 404)\n    except TableNotFound as e:\n        return False, _error([\"Table not found: {}\".format(e.table)], 404)\n    except RowNotFound as e:\n        return False, _error([\"Record not found: {}\".format(e.pk_values)], 404)\n\n    # Ensure user has permission to delete this row\n    if not await datasette.allowed(\n        action=permission,\n        resource=TableResource(database=resolved.db.name, table=resolved.table),\n        actor=request.actor,\n    ):\n        return False, _error([\"Permission denied\"], 403)\n\n    return True, resolved\n\n\nclass RowDeleteView(BaseView):\n    name = \"row-delete\"\n\n    def __init__(self, datasette):\n        self.ds = datasette\n\n    async def post(self, request):\n        ok, resolved = await _resolve_row_and_check_permission(\n            self.ds, request, \"delete-row\"\n        )\n        if not ok:\n            return resolved\n\n        # Delete table\n        def delete_row(conn):\n            sqlite_utils.Database(conn)[resolved.table].delete(resolved.pk_values)\n\n        try:\n            await resolved.db.execute_write_fn(delete_row, request=request)\n        except Exception as e:\n            return _error([str(e)], 500)\n\n        await self.ds.track_event(\n            DeleteRowEvent(\n                actor=request.actor,\n                database=resolved.db.name,\n                table=resolved.table,\n                pks=resolved.pk_values,\n            )\n        )\n\n        return Response.json({\"ok\": True}, status=200)\n\n\nclass RowUpdateView(BaseView):\n    name = \"row-update\"\n\n    def __init__(self, datasette):\n        self.ds = datasette\n\n    async def post(self, request):\n        ok, resolved = await _resolve_row_and_check_permission(\n            self.ds, request, \"update-row\"\n        )\n        if not ok:\n            return resolved\n\n        body = await request.post_body()\n        try:\n            data = json.loads(body)\n        except json.JSONDecodeError as e:\n            return _error([\"Invalid JSON: {}\".format(e)])\n\n        if not isinstance(data, dict):\n            return _error([\"JSON must be a dictionary\"])\n        if \"update\" not in data or not isinstance(data[\"update\"], dict):\n            return _error([\"JSON must contain an update dictionary\"])\n\n        invalid_keys = set(data.keys()) - {\"update\", \"return\", \"alter\"}\n        if invalid_keys:\n            return _error([\"Invalid keys: {}\".format(\", \".join(invalid_keys))])\n\n        update = data[\"update\"]\n\n        # Validate column types\n        from datasette.views.table import _validate_column_types\n\n        ct_errors = await _validate_column_types(\n            self.ds, resolved.db.name, resolved.table, [update]\n        )\n        if ct_errors:\n            return _error(ct_errors, 400)\n\n        alter = data.get(\"alter\")\n        if alter and not await self.ds.allowed(\n            action=\"alter-table\",\n            resource=TableResource(database=resolved.db.name, table=resolved.table),\n            actor=request.actor,\n        ):\n            return _error([\"Permission denied for alter-table\"], 403)\n\n        def update_row(conn):\n            sqlite_utils.Database(conn)[resolved.table].update(\n                resolved.pk_values, update, alter=alter\n            )\n\n        try:\n            await resolved.db.execute_write_fn(update_row, request=request)\n        except Exception as e:\n            return _error([str(e)], 400)\n\n        result = {\"ok\": True}\n        if data.get(\"return\"):\n            results = await resolved.db.execute(\n                resolved.sql, resolved.params, truncate=True\n            )\n            result[\"row\"] = results.dicts()[0]\n\n        await self.ds.track_event(\n            UpdateRowEvent(\n                actor=request.actor,\n                database=resolved.db.name,\n                table=resolved.table,\n                pks=resolved.pk_values,\n            )\n        )\n\n        return Response.json(result, status=200)\n"
  },
  {
    "path": "datasette/views/special.py",
    "content": "import json\nimport logging\nfrom datasette.events import LogoutEvent, LoginEvent, CreateTokenEvent\nfrom datasette.resources import DatabaseResource, TableResource\nfrom datasette.utils.asgi import Response, Forbidden\nfrom datasette.utils import (\n    actor_matches_allow,\n    add_cors_headers,\n    tilde_encode,\n    tilde_decode,\n)\nfrom .base import BaseView, View\nimport secrets\nimport urllib\n\nlogger = logging.getLogger(__name__)\n\n\ndef _resource_path(parent, child):\n    if parent is None:\n        return \"/\"\n    if child is None:\n        return f\"/{parent}\"\n    return f\"/{parent}/{child}\"\n\n\nclass JsonDataView(BaseView):\n    name = \"json_data\"\n    template = \"show_json.html\"  # Can be overridden in subclasses\n\n    def __init__(\n        self,\n        datasette,\n        filename,\n        data_callback,\n        needs_request=False,\n        permission=\"view-instance\",\n        template=None,\n    ):\n        self.ds = datasette\n        self.filename = filename\n        self.data_callback = data_callback\n        self.needs_request = needs_request\n        self.permission = permission\n        if template is not None:\n            self.template = template\n\n    async def get(self, request):\n        if self.permission:\n            await self.ds.ensure_permission(action=self.permission, actor=request.actor)\n        if self.needs_request:\n            data = self.data_callback(request)\n        else:\n            data = self.data_callback()\n\n        # Return JSON or HTML depending on format parameter\n        as_format = request.url_vars.get(\"format\")\n        if as_format:\n            headers = {}\n            if self.ds.cors:\n                add_cors_headers(headers)\n            return Response.json(data, headers=headers)\n        else:\n            context = {\n                \"filename\": self.filename,\n                \"data\": data,\n                \"data_json\": json.dumps(data, indent=4, default=repr),\n            }\n            # Add has_debug_permission if this view requires permissions-debug\n            if self.permission == \"permissions-debug\":\n                context[\"has_debug_permission\"] = True\n            return await self.render(\n                [self.template],\n                request=request,\n                context=context,\n            )\n\n\nclass PatternPortfolioView(View):\n    async def get(self, request, datasette):\n        await datasette.ensure_permission(action=\"view-instance\", actor=request.actor)\n        return Response.html(\n            await datasette.render_template(\n                \"patterns.html\",\n                request=request,\n                view_name=\"patterns\",\n            )\n        )\n\n\nclass AuthTokenView(BaseView):\n    name = \"auth_token\"\n    has_json_alternate = False\n\n    async def get(self, request):\n        # If already signed in as root, redirect\n        if request.actor and request.actor.get(\"id\") == \"root\":\n            return Response.redirect(self.ds.urls.instance())\n        token = request.args.get(\"token\") or \"\"\n        if not self.ds._root_token:\n            raise Forbidden(\"Root token has already been used\")\n        if secrets.compare_digest(token, self.ds._root_token):\n            self.ds._root_token = None\n            response = Response.redirect(self.ds.urls.instance())\n            root_actor = {\"id\": \"root\"}\n            self.ds.set_actor_cookie(response, root_actor)\n            await self.ds.track_event(LoginEvent(actor=root_actor))\n            return response\n        else:\n            raise Forbidden(\"Invalid token\")\n\n\nclass LogoutView(BaseView):\n    name = \"logout\"\n    has_json_alternate = False\n\n    async def get(self, request):\n        if not request.actor:\n            return Response.redirect(self.ds.urls.instance())\n        return await self.render(\n            [\"logout.html\"],\n            request,\n            {\"actor\": request.actor},\n        )\n\n    async def post(self, request):\n        response = Response.redirect(self.ds.urls.instance())\n        self.ds.delete_actor_cookie(response)\n        self.ds.add_message(request, \"You are now logged out\", self.ds.WARNING)\n        await self.ds.track_event(LogoutEvent(actor=request.actor))\n        return response\n\n\nclass PermissionsDebugView(BaseView):\n    name = \"permissions_debug\"\n    has_json_alternate = False\n\n    async def get(self, request):\n        await self.ds.ensure_permission(action=\"view-instance\", actor=request.actor)\n        await self.ds.ensure_permission(action=\"permissions-debug\", actor=request.actor)\n        filter_ = request.args.get(\"filter\") or \"all\"\n        permission_checks = list(reversed(self.ds._permission_checks))\n        if filter_ == \"exclude-yours\":\n            permission_checks = [\n                check\n                for check in permission_checks\n                if (check.actor or {}).get(\"id\") != request.actor[\"id\"]\n            ]\n        elif filter_ == \"only-yours\":\n            permission_checks = [\n                check\n                for check in permission_checks\n                if (check.actor or {}).get(\"id\") == request.actor[\"id\"]\n            ]\n        return await self.render(\n            [\"debug_permissions_playground.html\"],\n            request,\n            # list() avoids error if check is performed during template render:\n            {\n                \"permission_checks\": permission_checks,\n                \"filter\": filter_,\n                \"has_debug_permission\": True,\n                \"permissions\": [\n                    {\n                        \"name\": p.name,\n                        \"abbr\": p.abbr,\n                        \"description\": p.description,\n                        \"takes_parent\": p.takes_parent,\n                        \"takes_child\": p.takes_child,\n                    }\n                    for p in self.ds.actions.values()\n                ],\n            },\n        )\n\n    async def post(self, request):\n        await self.ds.ensure_permission(action=\"view-instance\", actor=request.actor)\n        await self.ds.ensure_permission(action=\"permissions-debug\", actor=request.actor)\n        form = await request.form()\n        actor = json.loads(form[\"actor\"])\n        permission = form[\"permission\"]\n        parent = form.get(\"resource_1\") or None\n        child = form.get(\"resource_2\") or None\n\n        response, status = await _check_permission_for_actor(\n            self.ds, permission, parent, child, actor\n        )\n        return Response.json(response, status=status)\n\n\nclass AllowedResourcesView(BaseView):\n    name = \"allowed\"\n    has_json_alternate = False\n\n    async def get(self, request):\n        await self.ds.refresh_schemas()\n\n        # Check if user has permissions-debug (to show sensitive fields)\n        has_debug_permission = await self.ds.allowed(\n            action=\"permissions-debug\", actor=request.actor\n        )\n\n        # Check if this is a request for JSON (has .json extension)\n        as_format = request.url_vars.get(\"format\")\n\n        if not as_format:\n            # Render the HTML form (even if query parameters are present)\n            # Put most common/interesting actions first\n            priority_actions = [\n                \"view-instance\",\n                \"view-database\",\n                \"view-table\",\n                \"view-query\",\n                \"execute-sql\",\n                \"insert-row\",\n                \"update-row\",\n                \"delete-row\",\n            ]\n            actions = list(self.ds.actions.keys())\n            # Priority actions first (in order), then remaining alphabetically\n            sorted_actions = [a for a in priority_actions if a in actions]\n            sorted_actions.extend(\n                sorted(a for a in actions if a not in priority_actions)\n            )\n\n            return await self.render(\n                [\"debug_allowed.html\"],\n                request,\n                {\n                    \"supported_actions\": sorted_actions,\n                    \"has_debug_permission\": has_debug_permission,\n                },\n            )\n\n        payload, status = await self._allowed_payload(request, has_debug_permission)\n        headers = {}\n        if self.ds.cors:\n            add_cors_headers(headers)\n        return Response.json(payload, status=status, headers=headers)\n\n    async def _allowed_payload(self, request, has_debug_permission):\n        action = request.args.get(\"action\")\n        if not action:\n            return {\"error\": \"action parameter is required\"}, 400\n        if action not in self.ds.actions:\n            return {\"error\": f\"Unknown action: {action}\"}, 404\n\n        actor = request.actor if isinstance(request.actor, dict) else None\n        actor_id = actor.get(\"id\") if actor else None\n        parent_filter = request.args.get(\"parent\")\n        child_filter = request.args.get(\"child\")\n        if child_filter and not parent_filter:\n            return {\"error\": \"parent must be provided when child is specified\"}, 400\n\n        try:\n            page = int(request.args.get(\"page\", \"1\"))\n            page_size = int(request.args.get(\"page_size\", \"50\"))\n        except ValueError:\n            return {\"error\": \"page and page_size must be integers\"}, 400\n        if page < 1:\n            return {\"error\": \"page must be >= 1\"}, 400\n        if page_size < 1:\n            return {\"error\": \"page_size must be >= 1\"}, 400\n        max_page_size = 200\n        if page_size > max_page_size:\n            page_size = max_page_size\n        offset = (page - 1) * page_size\n\n        # Use the simplified allowed_resources method\n        # Collect all resources with optional reasons for debugging\n        try:\n            allowed_rows = []\n            result = await self.ds.allowed_resources(\n                action=action,\n                actor=actor,\n                parent=parent_filter,\n                include_reasons=has_debug_permission,\n            )\n            async for resource in result.all():\n                parent_val = resource.parent\n                child_val = resource.child\n\n                # Build resource path\n                if parent_val is None:\n                    resource_path = \"/\"\n                elif child_val is None:\n                    resource_path = f\"/{parent_val}\"\n                else:\n                    resource_path = f\"/{parent_val}/{child_val}\"\n\n                row = {\n                    \"parent\": parent_val,\n                    \"child\": child_val,\n                    \"resource\": resource_path,\n                }\n\n                # Add reason if we have it (from include_reasons=True)\n                if has_debug_permission and hasattr(resource, \"reasons\"):\n                    row[\"reason\"] = resource.reasons\n\n                allowed_rows.append(row)\n        except Exception:\n            # If catalog tables don't exist yet, return empty results\n            return (\n                {\n                    \"action\": action,\n                    \"actor_id\": actor_id,\n                    \"page\": page,\n                    \"page_size\": page_size,\n                    \"total\": 0,\n                    \"items\": [],\n                },\n                200,\n            )\n\n        # Apply child filter if specified\n        if child_filter is not None:\n            allowed_rows = [row for row in allowed_rows if row[\"child\"] == child_filter]\n\n        # Pagination\n        total = len(allowed_rows)\n        paged_rows = allowed_rows[offset : offset + page_size]\n\n        # Items are already in the right format\n        items = paged_rows\n\n        def build_page_url(page_number):\n            pairs = []\n            for key in request.args:\n                if key in {\"page\", \"page_size\"}:\n                    continue\n                for value in request.args.getlist(key):\n                    pairs.append((key, value))\n            pairs.append((\"page\", str(page_number)))\n            pairs.append((\"page_size\", str(page_size)))\n            query = urllib.parse.urlencode(pairs)\n            return f\"{request.path}?{query}\"\n\n        response = {\n            \"action\": action,\n            \"actor_id\": actor_id,\n            \"page\": page,\n            \"page_size\": page_size,\n            \"total\": total,\n            \"items\": items,\n        }\n\n        if total > offset + page_size:\n            response[\"next_url\"] = build_page_url(page + 1)\n        if page > 1:\n            response[\"previous_url\"] = build_page_url(page - 1)\n\n        return response, 200\n\n\nclass PermissionRulesView(BaseView):\n    name = \"permission_rules\"\n    has_json_alternate = False\n\n    async def get(self, request):\n        await self.ds.ensure_permission(action=\"view-instance\", actor=request.actor)\n        await self.ds.ensure_permission(action=\"permissions-debug\", actor=request.actor)\n\n        # Check if this is a request for JSON (has .json extension)\n        as_format = request.url_vars.get(\"format\")\n\n        if not as_format:\n            # Render the HTML form (even if query parameters are present)\n            return await self.render(\n                [\"debug_rules.html\"],\n                request,\n                {\n                    \"sorted_actions\": sorted(self.ds.actions.keys()),\n                    \"has_debug_permission\": True,\n                },\n            )\n\n        # JSON API - action parameter is required\n        action = request.args.get(\"action\")\n        if not action:\n            return Response.json({\"error\": \"action parameter is required\"}, status=400)\n        if action not in self.ds.actions:\n            return Response.json({\"error\": f\"Unknown action: {action}\"}, status=404)\n\n        actor = request.actor if isinstance(request.actor, dict) else None\n\n        try:\n            page = int(request.args.get(\"page\", \"1\"))\n            page_size = int(request.args.get(\"page_size\", \"50\"))\n        except ValueError:\n            return Response.json(\n                {\"error\": \"page and page_size must be integers\"}, status=400\n            )\n        if page < 1:\n            return Response.json({\"error\": \"page must be >= 1\"}, status=400)\n        if page_size < 1:\n            return Response.json({\"error\": \"page_size must be >= 1\"}, status=400)\n        max_page_size = 200\n        if page_size > max_page_size:\n            page_size = max_page_size\n        offset = (page - 1) * page_size\n\n        from datasette.utils.actions_sql import build_permission_rules_sql\n\n        union_sql, union_params, restriction_sqls = await build_permission_rules_sql(\n            self.ds, actor, action\n        )\n        await self.ds.refresh_schemas()\n        db = self.ds.get_internal_database()\n\n        count_query = f\"\"\"\n        WITH rules AS (\n            {union_sql}\n        )\n        SELECT COUNT(*) AS count\n        FROM rules\n        \"\"\"\n        count_row = (await db.execute(count_query, union_params)).first()\n        total = count_row[\"count\"] if count_row else 0\n\n        data_query = f\"\"\"\n        WITH rules AS (\n            {union_sql}\n        )\n        SELECT parent, child, allow, reason, source_plugin\n        FROM rules\n        ORDER BY allow DESC, (parent IS NOT NULL), parent, child\n        LIMIT :limit OFFSET :offset\n        \"\"\"\n        params = {**union_params, \"limit\": page_size, \"offset\": offset}\n        rows = await db.execute(data_query, params)\n\n        items = []\n        for row in rows:\n            parent = row[\"parent\"]\n            child = row[\"child\"]\n            items.append(\n                {\n                    \"parent\": parent,\n                    \"child\": child,\n                    \"resource\": _resource_path(parent, child),\n                    \"allow\": row[\"allow\"],\n                    \"reason\": row[\"reason\"],\n                    \"source_plugin\": row[\"source_plugin\"],\n                }\n            )\n\n        def build_page_url(page_number):\n            pairs = []\n            for key in request.args:\n                if key in {\"page\", \"page_size\"}:\n                    continue\n                for value in request.args.getlist(key):\n                    pairs.append((key, value))\n            pairs.append((\"page\", str(page_number)))\n            pairs.append((\"page_size\", str(page_size)))\n            query = urllib.parse.urlencode(pairs)\n            return f\"{request.path}?{query}\"\n\n        response = {\n            \"action\": action,\n            \"actor_id\": (actor or {}).get(\"id\") if actor else None,\n            \"page\": page,\n            \"page_size\": page_size,\n            \"total\": total,\n            \"items\": items,\n        }\n\n        if total > offset + page_size:\n            response[\"next_url\"] = build_page_url(page + 1)\n        if page > 1:\n            response[\"previous_url\"] = build_page_url(page - 1)\n\n        headers = {}\n        if self.ds.cors:\n            add_cors_headers(headers)\n        return Response.json(response, headers=headers)\n\n\nasync def _check_permission_for_actor(ds, action, parent, child, actor):\n    \"\"\"Shared logic for checking permissions. Returns a dict with check results.\"\"\"\n    if action not in ds.actions:\n        return {\"error\": f\"Unknown action: {action}\"}, 404\n\n    if child and not parent:\n        return {\"error\": \"parent is required when child is provided\"}, 400\n\n    # Use the action's properties to create the appropriate resource object\n    action_obj = ds.actions.get(action)\n    if not action_obj:\n        return {\"error\": f\"Unknown action: {action}\"}, 400\n\n    # Global actions (no resource_class) don't have a resource\n    if action_obj.resource_class is None:\n        resource_obj = None\n    elif action_obj.takes_parent and action_obj.takes_child:\n        # Child-level resource (e.g., TableResource, QueryResource)\n        resource_obj = action_obj.resource_class(database=parent, table=child)\n    elif action_obj.takes_parent:\n        # Parent-level resource (e.g., DatabaseResource)\n        resource_obj = action_obj.resource_class(database=parent)\n    else:\n        # This shouldn't happen given validation in Action.__post_init__\n        return {\"error\": f\"Invalid action configuration: {action}\"}, 500\n\n    allowed = await ds.allowed(action=action, resource=resource_obj, actor=actor)\n\n    response = {\n        \"action\": action,\n        \"allowed\": bool(allowed),\n        \"resource\": {\n            \"parent\": parent,\n            \"child\": child,\n            \"path\": _resource_path(parent, child),\n        },\n    }\n\n    if actor and \"id\" in actor:\n        response[\"actor_id\"] = actor[\"id\"]\n\n    return response, 200\n\n\nclass PermissionCheckView(BaseView):\n    name = \"permission_check\"\n    has_json_alternate = False\n\n    async def get(self, request):\n        await self.ds.ensure_permission(action=\"permissions-debug\", actor=request.actor)\n        as_format = request.url_vars.get(\"format\")\n\n        if not as_format:\n            return await self.render(\n                [\"debug_check.html\"],\n                request,\n                {\n                    \"sorted_actions\": sorted(self.ds.actions.keys()),\n                    \"has_debug_permission\": True,\n                },\n            )\n\n        # JSON API - action parameter is required\n        action = request.args.get(\"action\")\n        if not action:\n            return Response.json({\"error\": \"action parameter is required\"}, status=400)\n\n        parent = request.args.get(\"parent\")\n        child = request.args.get(\"child\")\n\n        response, status = await _check_permission_for_actor(\n            self.ds, action, parent, child, request.actor\n        )\n        return Response.json(response, status=status)\n\n\nclass AllowDebugView(BaseView):\n    name = \"allow_debug\"\n    has_json_alternate = False\n\n    async def get(self, request):\n        errors = []\n        actor_input = request.args.get(\"actor\") or '{\"id\": \"root\"}'\n        try:\n            actor = json.loads(actor_input)\n            actor_input = json.dumps(actor, indent=4)\n        except json.decoder.JSONDecodeError as ex:\n            errors.append(f\"Actor JSON error: {ex}\")\n        allow_input = request.args.get(\"allow\") or '{\"id\": \"*\"}'\n        try:\n            allow = json.loads(allow_input)\n            allow_input = json.dumps(allow, indent=4)\n        except json.decoder.JSONDecodeError as ex:\n            errors.append(f\"Allow JSON error: {ex}\")\n\n        result = None\n        if not errors:\n            result = str(actor_matches_allow(actor, allow))\n\n        return await self.render(\n            [\"allow_debug.html\"],\n            request,\n            {\n                \"result\": result,\n                \"error\": \"\\n\\n\".join(errors) if errors else \"\",\n                \"actor_input\": actor_input,\n                \"allow_input\": allow_input,\n                \"has_debug_permission\": await self.ds.allowed(\n                    action=\"permissions-debug\", actor=request.actor\n                ),\n            },\n        )\n\n\nclass MessagesDebugView(BaseView):\n    name = \"messages_debug\"\n    has_json_alternate = False\n\n    async def get(self, request):\n        await self.ds.ensure_permission(action=\"view-instance\", actor=request.actor)\n        return await self.render([\"messages_debug.html\"], request)\n\n    async def post(self, request):\n        await self.ds.ensure_permission(action=\"view-instance\", actor=request.actor)\n        form = await request.form()\n        message = form.get(\"message\", \"\")\n        message_type = form.get(\"message_type\") or \"INFO\"\n        assert message_type in (\"INFO\", \"WARNING\", \"ERROR\", \"all\")\n        datasette = self.ds\n        if message_type == \"all\":\n            datasette.add_message(request, message, datasette.INFO)\n            datasette.add_message(request, message, datasette.WARNING)\n            datasette.add_message(request, message, datasette.ERROR)\n        else:\n            datasette.add_message(request, message, getattr(datasette, message_type))\n        return Response.redirect(self.ds.urls.instance())\n\n\nclass CreateTokenView(BaseView):\n    name = \"create_token\"\n    has_json_alternate = False\n\n    def check_permission(self, request):\n        if not self.ds.setting(\"allow_signed_tokens\"):\n            raise Forbidden(\"Signed tokens are not enabled for this Datasette instance\")\n        if not request.actor:\n            raise Forbidden(\"You must be logged in to create a token\")\n        if not request.actor.get(\"id\"):\n            raise Forbidden(\n                \"You must be logged in as an actor with an ID to create a token\"\n            )\n        if request.actor.get(\"token\"):\n            raise Forbidden(\n                \"Token authentication cannot be used to create additional tokens\"\n            )\n\n    async def shared(self, request):\n        self.check_permission(request)\n        # Build list of databases and tables the user has permission to view\n        db_page = await self.ds.allowed_resources(\"view-database\", request.actor)\n        allowed_databases = [r async for r in db_page.all()]\n\n        table_page = await self.ds.allowed_resources(\"view-table\", request.actor)\n        allowed_tables = [r async for r in table_page.all()]\n\n        # Build database -> tables mapping\n        database_with_tables = []\n        for db_resource in allowed_databases:\n            database_name = db_resource.parent\n            if database_name == \"_memory\":\n                continue\n\n            # Find tables for this database\n            tables = []\n            for table_resource in allowed_tables:\n                if table_resource.parent == database_name:\n                    tables.append(\n                        {\n                            \"name\": table_resource.child,\n                            \"encoded\": tilde_encode(table_resource.child),\n                        }\n                    )\n\n            database_with_tables.append(\n                {\n                    \"name\": database_name,\n                    \"encoded\": tilde_encode(database_name),\n                    \"tables\": tables,\n                }\n            )\n        return {\n            \"actor\": request.actor,\n            \"all_actions\": self.ds.actions.keys(),\n            \"database_actions\": [\n                key for key, value in self.ds.actions.items() if value.takes_parent\n            ],\n            \"child_actions\": [\n                key for key, value in self.ds.actions.items() if value.takes_child\n            ],\n            \"database_with_tables\": database_with_tables,\n        }\n\n    async def get(self, request):\n        self.check_permission(request)\n        return await self.render(\n            [\"create_token.html\"], request, await self.shared(request)\n        )\n\n    async def post(self, request):\n        self.check_permission(request)\n        form = await request.form()\n        errors = []\n        expires_after = None\n        if form.get(\"expire_type\"):\n            duration_string = form.get(\"expire_duration\")\n            if (\n                not duration_string\n                or not duration_string.isdigit()\n                or not int(duration_string) > 0\n            ):\n                errors.append(\"Invalid expire duration\")\n            else:\n                unit = form[\"expire_type\"]\n                if unit == \"minutes\":\n                    expires_after = int(duration_string) * 60\n                elif unit == \"hours\":\n                    expires_after = int(duration_string) * 60 * 60\n                elif unit == \"days\":\n                    expires_after = int(duration_string) * 60 * 60 * 24\n                else:\n                    errors.append(\"Invalid expire duration unit\")\n\n        # Are there any restrictions?\n        from datasette.tokens import TokenRestrictions\n\n        restrictions = TokenRestrictions()\n\n        for key in form:\n            if key.startswith(\"all:\") and key.count(\":\") == 1:\n                restrictions.allow_all(key.split(\":\")[1])\n            elif key.startswith(\"database:\") and key.count(\":\") == 2:\n                bits = key.split(\":\")\n                restrictions.allow_database(tilde_decode(bits[1]), bits[2])\n            elif key.startswith(\"resource:\") and key.count(\":\") == 3:\n                bits = key.split(\":\")\n                restrictions.allow_resource(\n                    tilde_decode(bits[1]), tilde_decode(bits[2]), bits[3]\n                )\n\n        token = await self.ds.create_token(\n            request.actor[\"id\"],\n            expires_after=expires_after,\n            restrictions=restrictions,\n            handler=\"signed\",\n        )\n        token_bits = self.ds.unsign(token[len(\"dstok_\") :], namespace=\"token\")\n        await self.ds.track_event(\n            CreateTokenEvent(\n                actor=request.actor,\n                expires_after=expires_after,\n                restrict_all=restrictions.all,\n                restrict_database=restrictions.database,\n                restrict_resource=restrictions.resource,\n            )\n        )\n        context = await self.shared(request)\n        context.update({\"errors\": errors, \"token\": token, \"token_bits\": token_bits})\n        return await self.render([\"create_token.html\"], request, context)\n\n\nclass ApiExplorerView(BaseView):\n    name = \"api_explorer\"\n    has_json_alternate = False\n\n    async def example_links(self, request):\n        databases = []\n        for name, db in self.ds.databases.items():\n            database_visible, _ = await self.ds.check_visibility(\n                request.actor,\n                action=\"view-database\",\n                resource=DatabaseResource(database=name),\n            )\n            if not database_visible:\n                continue\n            tables = []\n            table_names = await db.table_names()\n            for table in table_names:\n                visible, _ = await self.ds.check_visibility(\n                    request.actor,\n                    action=\"view-table\",\n                    resource=TableResource(database=name, table=table),\n                )\n                if not visible:\n                    continue\n                table_links = []\n                tables.append({\"name\": table, \"links\": table_links})\n                table_links.append(\n                    {\n                        \"label\": \"Get rows for {}\".format(table),\n                        \"method\": \"GET\",\n                        \"path\": self.ds.urls.table(name, table, format=\"json\"),\n                    }\n                )\n                # If not mutable don't show any write APIs\n                if not db.is_mutable:\n                    continue\n\n                if await self.ds.allowed(\n                    action=\"insert-row\",\n                    resource=TableResource(database=name, table=table),\n                    actor=request.actor,\n                ):\n                    pks = await db.primary_keys(table)\n                    table_links.extend(\n                        [\n                            {\n                                \"path\": self.ds.urls.table(name, table) + \"/-/insert\",\n                                \"method\": \"POST\",\n                                \"label\": \"Insert rows into {}\".format(table),\n                                \"json\": {\n                                    \"rows\": [\n                                        {\n                                            column: None\n                                            for column in await db.table_columns(table)\n                                            if column not in pks\n                                        }\n                                    ]\n                                },\n                            },\n                            {\n                                \"path\": self.ds.urls.table(name, table) + \"/-/upsert\",\n                                \"method\": \"POST\",\n                                \"label\": \"Upsert rows into {}\".format(table),\n                                \"json\": {\n                                    \"rows\": [\n                                        {\n                                            column: None\n                                            for column in await db.table_columns(table)\n                                            if column not in pks\n                                        }\n                                    ]\n                                },\n                            },\n                        ]\n                    )\n                if await self.ds.allowed(\n                    action=\"drop-table\",\n                    resource=TableResource(database=name, table=table),\n                    actor=request.actor,\n                ):\n                    table_links.append(\n                        {\n                            \"path\": self.ds.urls.table(name, table) + \"/-/drop\",\n                            \"label\": \"Drop table {}\".format(table),\n                            \"json\": {\"confirm\": False},\n                            \"method\": \"POST\",\n                        }\n                    )\n            database_links = []\n            if (\n                await self.ds.allowed(\n                    action=\"create-table\",\n                    resource=DatabaseResource(database=name),\n                    actor=request.actor,\n                )\n                and db.is_mutable\n            ):\n                database_links.append(\n                    {\n                        \"path\": self.ds.urls.database(name) + \"/-/create\",\n                        \"label\": \"Create table in {}\".format(name),\n                        \"json\": {\n                            \"table\": \"new_table\",\n                            \"columns\": [\n                                {\"name\": \"id\", \"type\": \"integer\"},\n                                {\"name\": \"name\", \"type\": \"text\"},\n                            ],\n                            \"pk\": \"id\",\n                        },\n                        \"method\": \"POST\",\n                    }\n                )\n            if database_links or tables:\n                databases.append(\n                    {\n                        \"name\": name,\n                        \"links\": database_links,\n                        \"tables\": tables,\n                    }\n                )\n        # Sort so that mutable databases are first\n        databases.sort(key=lambda d: not self.ds.databases[d[\"name\"]].is_mutable)\n        return databases\n\n    async def get(self, request):\n        visible, private = await self.ds.check_visibility(\n            request.actor,\n            action=\"view-instance\",\n        )\n        if not visible:\n            raise Forbidden(\"You do not have permission to view this instance\")\n\n        def api_path(link):\n            return \"/-/api#{}\".format(\n                urllib.parse.urlencode(\n                    {\n                        key: json.dumps(value, indent=2) if key == \"json\" else value\n                        for key, value in link.items()\n                        if key in (\"path\", \"method\", \"json\")\n                    }\n                )\n            )\n\n        return await self.render(\n            [\"api_explorer.html\"],\n            request,\n            {\n                \"example_links\": await self.example_links(request),\n                \"api_path\": api_path,\n                \"private\": private,\n            },\n        )\n\n\nclass TablesView(BaseView):\n    \"\"\"\n    Simple endpoint that uses the new allowed_resources() API.\n    Returns JSON list of all tables the actor can view.\n\n    Supports ?q=foo+bar to filter tables matching .*foo.*bar.* pattern,\n    ordered by shortest name first.\n    \"\"\"\n\n    name = \"tables\"\n    has_json_alternate = False\n\n    async def get(self, request):\n        # Get search query parameter\n        q = request.args.get(\"q\", \"\").strip()\n\n        # Get SQL for allowed resources using the permission system\n        permission_sql, params = await self.ds.allowed_resources_sql(\n            action=\"view-table\", actor=request.actor\n        )\n\n        # Build query based on whether we have a search query\n        if q:\n            # Build SQL LIKE pattern from search terms\n            # Split search terms by whitespace and build pattern: %term1%term2%term3%\n            terms = q.split()\n            pattern = \"%\" + \"%\".join(terms) + \"%\"\n\n            # Build query with CTE to filter by search pattern\n            sql = f\"\"\"\n            WITH allowed_tables AS (\n                {permission_sql}\n            )\n            SELECT parent, child\n            FROM allowed_tables\n            WHERE child LIKE :pattern COLLATE NOCASE\n            ORDER BY length(child), child\n            \"\"\"\n            all_params = {**params, \"pattern\": pattern}\n        else:\n            # No search query - return all tables, ordered by name\n            # Fetch 101 to detect if we need to truncate\n            sql = f\"\"\"\n            WITH allowed_tables AS (\n                {permission_sql}\n            )\n            SELECT parent, child\n            FROM allowed_tables\n            ORDER BY parent, child\n            LIMIT 101\n            \"\"\"\n            all_params = params\n\n        # Execute against internal database\n        result = await self.ds.get_internal_database().execute(sql, all_params)\n\n        # Build response with truncation\n        rows = list(result.rows)\n        truncated = len(rows) > 100\n        if truncated:\n            rows = rows[:100]\n\n        matches = [\n            {\n                \"name\": f\"{row['parent']}: {row['child']}\",\n                \"url\": self.ds.urls.table(row[\"parent\"], row[\"child\"]),\n            }\n            for row in rows\n        ]\n\n        return Response.json({\"matches\": matches, \"truncated\": truncated})\n\n\nclass SchemaBaseView(BaseView):\n    \"\"\"Base class for schema views with common response formatting.\"\"\"\n\n    has_json_alternate = False\n\n    async def get_database_schema(self, database_name):\n        \"\"\"Get schema SQL for a database.\"\"\"\n        db = self.ds.databases[database_name]\n        result = await db.execute(\n            \"select group_concat(sql, ';' || CHAR(10)) as schema from sqlite_master where sql is not null\"\n        )\n        row = result.first()\n        return row[\"schema\"] if row and row[\"schema\"] else \"\"\n\n    def format_json_response(self, data):\n        \"\"\"Format data as JSON response with CORS headers if needed.\"\"\"\n        headers = {}\n        if self.ds.cors:\n            add_cors_headers(headers)\n        return Response.json(data, headers=headers)\n\n    def format_error_response(self, error_message, format_, status=404):\n        \"\"\"Format error response based on requested format.\"\"\"\n        if format_ == \"json\":\n            headers = {}\n            if self.ds.cors:\n                add_cors_headers(headers)\n            return Response.json(\n                {\"ok\": False, \"error\": error_message}, status=status, headers=headers\n            )\n        else:\n            return Response.text(error_message, status=status)\n\n    def format_markdown_response(self, heading, schema):\n        \"\"\"Format schema as Markdown response.\"\"\"\n        md_output = f\"# {heading}\\n\\n```sql\\n{schema}\\n```\\n\"\n        return Response.text(\n            md_output, headers={\"content-type\": \"text/markdown; charset=utf-8\"}\n        )\n\n    async def format_html_response(\n        self, request, schemas, is_instance=False, table_name=None\n    ):\n        \"\"\"Format schema as HTML response.\"\"\"\n        context = {\n            \"schemas\": schemas,\n            \"is_instance\": is_instance,\n        }\n        if table_name:\n            context[\"table_name\"] = table_name\n        return await self.render([\"schema.html\"], request=request, context=context)\n\n\nclass InstanceSchemaView(SchemaBaseView):\n    \"\"\"\n    Displays schema for all databases in the instance.\n    Supports HTML, JSON, and Markdown formats.\n    \"\"\"\n\n    name = \"instance_schema\"\n\n    async def get(self, request):\n        format_ = request.url_vars.get(\"format\") or \"html\"\n\n        # Get all databases the actor can view\n        allowed_databases_page = await self.ds.allowed_resources(\n            \"view-database\",\n            request.actor,\n        )\n        allowed_databases = [r.parent async for r in allowed_databases_page.all()]\n\n        # Get schema for each database\n        schemas = []\n        for database_name in allowed_databases:\n            schema = await self.get_database_schema(database_name)\n            schemas.append({\"database\": database_name, \"schema\": schema})\n\n        if format_ == \"json\":\n            return self.format_json_response({\"schemas\": schemas})\n        elif format_ == \"md\":\n            md_parts = [\n                f\"# Schema for {item['database']}\\n\\n```sql\\n{item['schema']}\\n```\"\n                for item in schemas\n            ]\n            return Response.text(\n                \"\\n\\n\".join(md_parts),\n                headers={\"content-type\": \"text/markdown; charset=utf-8\"},\n            )\n        else:\n            return await self.format_html_response(request, schemas, is_instance=True)\n\n\nclass DatabaseSchemaView(SchemaBaseView):\n    \"\"\"\n    Displays schema for a specific database.\n    Supports HTML, JSON, and Markdown formats.\n    \"\"\"\n\n    name = \"database_schema\"\n\n    async def get(self, request):\n        database_name = request.url_vars[\"database\"]\n        format_ = request.url_vars.get(\"format\") or \"html\"\n\n        # Check if database exists\n        if database_name not in self.ds.databases:\n            return self.format_error_response(\"Database not found\", format_)\n\n        # Check view-database permission\n        await self.ds.ensure_permission(\n            action=\"view-database\",\n            resource=DatabaseResource(database=database_name),\n            actor=request.actor,\n        )\n\n        schema = await self.get_database_schema(database_name)\n\n        if format_ == \"json\":\n            return self.format_json_response(\n                {\"database\": database_name, \"schema\": schema}\n            )\n        elif format_ == \"md\":\n            return self.format_markdown_response(f\"Schema for {database_name}\", schema)\n        else:\n            schemas = [{\"database\": database_name, \"schema\": schema}]\n            return await self.format_html_response(request, schemas)\n\n\nclass TableSchemaView(SchemaBaseView):\n    \"\"\"\n    Displays schema for a specific table.\n    Supports HTML, JSON, and Markdown formats.\n    \"\"\"\n\n    name = \"table_schema\"\n\n    async def get(self, request):\n        database_name = request.url_vars[\"database\"]\n        table_name = request.url_vars[\"table\"]\n        format_ = request.url_vars.get(\"format\") or \"html\"\n\n        # Check view-table permission\n        await self.ds.ensure_permission(\n            action=\"view-table\",\n            resource=TableResource(database=database_name, table=table_name),\n            actor=request.actor,\n        )\n\n        # Get schema for the table\n        db = self.ds.databases[database_name]\n        result = await db.execute(\n            \"select sql from sqlite_master where name = ? and sql is not null\",\n            [table_name],\n        )\n        row = result.first()\n\n        # Return 404 if table doesn't exist\n        if not row or not row[\"sql\"]:\n            return self.format_error_response(\"Table not found\", format_)\n\n        schema = row[\"sql\"]\n\n        if format_ == \"json\":\n            return self.format_json_response(\n                {\"database\": database_name, \"table\": table_name, \"schema\": schema}\n            )\n        elif format_ == \"md\":\n            return self.format_markdown_response(\n                f\"Schema for {database_name}.{table_name}\", schema\n            )\n        else:\n            schemas = [{\"database\": database_name, \"schema\": schema}]\n            return await self.format_html_response(\n                request, schemas, table_name=table_name\n            )\n"
  },
  {
    "path": "datasette/views/table.py",
    "content": "import asyncio\nimport itertools\nimport json\nimport urllib\n\nfrom asyncinject import Registry\nimport markupsafe\n\nfrom datasette.plugins import pm\nfrom datasette.database import QueryInterrupted\nfrom datasette.events import (\n    AlterTableEvent,\n    DropTableEvent,\n    InsertRowsEvent,\n    UpsertRowsEvent,\n)\nfrom datasette import tracer\nfrom datasette.resources import DatabaseResource, TableResource\nfrom datasette.utils import (\n    add_cors_headers,\n    await_me_maybe,\n    call_with_supported_arguments,\n    CustomRow,\n    append_querystring,\n    compound_keys_after_sql,\n    format_bytes,\n    make_slot_function,\n    tilde_encode,\n    escape_sqlite,\n    filters_should_redirect,\n    is_url,\n    path_from_row_pks,\n    path_with_added_args,\n    path_with_format,\n    path_with_removed_args,\n    path_with_replaced_args,\n    to_css_class,\n    truncate_url,\n    urlsafe_components,\n    value_as_boolean,\n    InvalidSql,\n    sqlite3,\n)\nfrom datasette.utils.asgi import BadRequest, Forbidden, NotFound, Response\nfrom datasette.filters import Filters\nimport sqlite_utils\nfrom .base import BaseView, DatasetteError, _error, stream_csv\nfrom .database import QueryView\n\nLINK_WITH_LABEL = (\n    '{label} {id}'\n)\nLINK_WITH_VALUE = '{id}'\n\n\nclass Row:\n    def __init__(self, cells):\n        self.cells = cells\n\n    def __iter__(self):\n        return iter(self.cells)\n\n    def __getitem__(self, key):\n        for cell in self.cells:\n            if cell[\"column\"] == key:\n                return cell[\"raw\"]\n        raise KeyError\n\n    def display(self, key):\n        for cell in self.cells:\n            if cell[\"column\"] == key:\n                return cell[\"value\"]\n        return None\n\n    def __str__(self):\n        d = {\n            key: self[key]\n            for key in [\n                c[\"column\"] for c in self.cells if not c.get(\"is_special_link_column\")\n            ]\n        }\n        return json.dumps(d, default=repr, indent=2)\n\n\nasync def run_sequential(*args):\n    # This used to be swappable for asyncio.gather() to run things in\n    # parallel, but this lead to hard-to-debug locking issues with\n    # in-memory databases: https://github.com/simonw/datasette/issues/2189\n    results = []\n    for fn in args:\n        results.append(await fn)\n    return results\n\n\ndef _redirect(datasette, request, path, forward_querystring=True, remove_args=None):\n    if request.query_string and \"?\" not in path and forward_querystring:\n        path = f\"{path}?{request.query_string}\"\n    if remove_args:\n        path = path_with_removed_args(request, remove_args, path=path)\n    r = Response.redirect(path)\n    r.headers[\"Link\"] = f\"<{path}>; rel=preload\"\n    if datasette.cors:\n        add_cors_headers(r.headers)\n    return r\n\n\nasync def _redirect_if_needed(datasette, request, resolved):\n    # Handle ?_filter_column\n    redirect_params = filters_should_redirect(request.args)\n    if redirect_params:\n        return _redirect(\n            datasette,\n            request,\n            datasette.urls.path(path_with_added_args(request, redirect_params)),\n            forward_querystring=False,\n        )\n\n    # If ?_sort_by_desc=on (from checkbox) redirect to _sort_desc=(_sort)\n    if \"_sort_by_desc\" in request.args:\n        return _redirect(\n            datasette,\n            request,\n            datasette.urls.path(\n                path_with_added_args(\n                    request,\n                    {\n                        \"_sort_desc\": request.args.get(\"_sort\"),\n                        \"_sort_by_desc\": None,\n                        \"_sort\": None,\n                    },\n                )\n            ),\n            forward_querystring=False,\n        )\n\n\nasync def _validate_column_types(datasette, database_name, table_name, rows):\n    \"\"\"Validate row values against assigned column types. Returns list of error strings.\"\"\"\n    ct_map = await datasette.get_column_types(database_name, table_name)\n    if not ct_map:\n        return []\n    errors = []\n    for row in rows:\n        for col_name, ct in ct_map.items():\n            if col_name not in row:\n                continue\n            error = await ct.validate(row[col_name], datasette)\n            if error:\n                errors.append(f\"{col_name}: {error}\")\n    return errors\n\n\nasync def display_columns_and_rows(\n    datasette,\n    database_name,\n    table_name,\n    description,\n    rows,\n    link_column=False,\n    truncate_cells=0,\n    sortable_columns=None,\n    request=None,\n):\n    \"\"\"Returns columns, rows for specified table - including fancy foreign key treatment\"\"\"\n    sortable_columns = sortable_columns or set()\n    db = datasette.databases[database_name]\n    column_descriptions = dict(\n        await datasette.get_internal_database().execute(\n            \"\"\"\n          SELECT\n            column_name,\n            value\n          FROM metadata_columns\n          WHERE database_name = ?\n            AND resource_name = ?\n            AND key = 'description'\n        \"\"\",\n            [database_name, table_name],\n        )\n    )\n\n    # Look up column types for this table\n    column_types_map = await datasette.get_column_types(database_name, table_name)\n\n    column_details = {\n        col.name: col for col in await db.table_column_details(table_name)\n    }\n    pks = await db.primary_keys(table_name)\n    pks_for_display = pks\n    if not pks_for_display:\n        pks_for_display = [\"rowid\"]\n\n    columns = []\n    for r in description:\n        if r[0] == \"rowid\" and \"rowid\" not in column_details:\n            type_ = \"integer\"\n            notnull = 0\n        else:\n            type_ = column_details[r[0]].type\n            notnull = column_details[r[0]].notnull\n        col_dict = {\n            \"name\": r[0],\n            \"sortable\": r[0] in sortable_columns,\n            \"is_pk\": r[0] in pks_for_display,\n            \"type\": type_,\n            \"notnull\": notnull,\n            \"description\": column_descriptions.get(r[0]),\n            \"column_type\": None,\n            \"column_type_config\": None,\n        }\n        ct = column_types_map.get(r[0])\n        if ct:\n            col_dict[\"column_type\"] = ct.name\n            col_dict[\"column_type_config\"] = ct.config\n        columns.append(col_dict)\n\n    column_to_foreign_key_table = {\n        fk[\"column\"]: fk[\"other_table\"]\n        for fk in await db.foreign_keys_for_table(table_name)\n    }\n\n    cell_rows = []\n    base_url = datasette.setting(\"base_url\")\n    for row in rows:\n        cells = []\n        # Unless we are a view, the first column is a link - either to the rowid\n        # or to the simple or compound primary key\n        if link_column:\n            is_special_link_column = len(pks) != 1\n            pk_path = path_from_row_pks(row, pks, not pks, False)\n            cells.append(\n                {\n                    \"column\": pks[0] if len(pks) == 1 else \"Link\",\n                    \"value_type\": \"pk\",\n                    \"is_special_link_column\": is_special_link_column,\n                    \"raw\": pk_path,\n                    \"value\": markupsafe.Markup(\n                        '{flat_pks}'.format(\n                            table_path=datasette.urls.table(database_name, table_name),\n                            flat_pks=str(markupsafe.escape(pk_path)),\n                            flat_pks_quoted=path_from_row_pks(row, pks, not pks),\n                        )\n                    ),\n                }\n            )\n\n        for value, column_dict in zip(row, columns):\n            column = column_dict[\"name\"]\n            if link_column and len(pks) == 1 and column == pks[0]:\n                # If there's a simple primary key, don't repeat the value as it's\n                # already shown in the link column.\n                continue\n\n            # First try column type render_cell, then plugins\n            # pylint: disable=no-member\n            plugin_display_value = None\n            ct = column_types_map.get(column)\n            if ct:\n                candidate = await ct.render_cell(\n                    value=value,\n                    column=column,\n                    table=table_name,\n                    database=database_name,\n                    datasette=datasette,\n                    request=request,\n                )\n                if candidate is not None:\n                    plugin_display_value = candidate\n            if plugin_display_value is None:\n                for candidate in pm.hook.render_cell(\n                    row=row,\n                    value=value,\n                    column=column,\n                    table=table_name,\n                    pks=pks_for_display,\n                    database=database_name,\n                    datasette=datasette,\n                    request=request,\n                    column_type=ct,\n                ):\n                    candidate = await await_me_maybe(candidate)\n                    if candidate is not None:\n                        plugin_display_value = candidate\n                        break\n            if plugin_display_value:\n                display_value = plugin_display_value\n            elif isinstance(value, bytes):\n                formatted = format_bytes(len(value))\n                display_value = markupsafe.Markup(\n                    '<Binary: {:,} byte{}>'.format(\n                        datasette.urls.row_blob(\n                            database_name,\n                            table_name,\n                            path_from_row_pks(row, pks, not pks),\n                            column,\n                        ),\n                        (\n                            ' title=\"{}\"'.format(formatted)\n                            if \"bytes\" not in formatted\n                            else \"\"\n                        ),\n                        len(value),\n                        \"\" if len(value) == 1 else \"s\",\n                    )\n                )\n            elif isinstance(value, dict):\n                # It's an expanded foreign key - display link to other row\n                label = value[\"label\"]\n                value = value[\"value\"]\n                # The table we link to depends on the column\n                other_table = column_to_foreign_key_table[column]\n                link_template = LINK_WITH_LABEL if (label != value) else LINK_WITH_VALUE\n                display_value = markupsafe.Markup(\n                    link_template.format(\n                        database=tilde_encode(database_name),\n                        base_url=base_url,\n                        table=tilde_encode(other_table),\n                        link_id=tilde_encode(str(value)),\n                        id=str(markupsafe.escape(value)),\n                        label=str(markupsafe.escape(label)) or \"-\",\n                    )\n                )\n            elif value in (\"\", None):\n                display_value = markupsafe.Markup(\" \")\n            elif is_url(str(value).strip()):\n                display_value = markupsafe.Markup(\n                    '{truncated_url}'.format(\n                        url=markupsafe.escape(value.strip()),\n                        truncated_url=markupsafe.escape(\n                            truncate_url(value.strip(), truncate_cells)\n                        ),\n                    )\n                )\n            else:\n                display_value = str(value)\n                if truncate_cells and len(display_value) > truncate_cells:\n                    display_value = display_value[:truncate_cells] + \"\\u2026\"\n\n            cells.append(\n                {\n                    \"column\": column,\n                    \"value\": display_value,\n                    \"raw\": value,\n                    \"value_type\": (\n                        \"none\" if value is None else str(type(value).__name__)\n                    ),\n                }\n            )\n        cell_rows.append(Row(cells))\n\n    if link_column:\n        # Add the link column header.\n        # If it's a simple primary key, we have to remove and re-add that column name at\n        # the beginning of the header row.\n        first_column = None\n        if len(pks) == 1:\n            columns = [col for col in columns if col[\"name\"] != pks[0]]\n            first_column = {\n                \"name\": pks[0],\n                \"sortable\": len(pks) == 1,\n                \"is_pk\": True,\n                \"type\": column_details[pks[0]].type,\n                \"notnull\": column_details[pks[0]].notnull,\n            }\n        else:\n            first_column = {\n                \"name\": \"Link\",\n                \"sortable\": False,\n                \"is_pk\": False,\n                \"type\": \"\",\n                \"notnull\": 0,\n                \"is_special_link_column\": True,\n            }\n        columns = [first_column] + columns\n    return columns, cell_rows\n\n\nclass TableInsertView(BaseView):\n    name = \"table-insert\"\n\n    def __init__(self, datasette):\n        self.ds = datasette\n\n    async def _validate_data(self, request, db, table_name, pks, upsert):\n        errors = []\n\n        pks_list = []\n        if isinstance(pks, str):\n            pks_list = [pks]\n        else:\n            pks_list = list(pks)\n\n        if not pks_list:\n            pks_list = [\"rowid\"]\n\n        def _errors(errors):\n            return None, errors, {}\n\n        if not request.headers.get(\"content-type\").startswith(\"application/json\"):\n            # TODO: handle form-encoded data\n            return _errors([\"Invalid content-type, must be application/json\"])\n        body = await request.post_body()\n        try:\n            data = json.loads(body)\n        except json.JSONDecodeError as e:\n            return _errors([\"Invalid JSON: {}\".format(e)])\n        if not isinstance(data, dict):\n            return _errors([\"JSON must be a dictionary\"])\n        keys = data.keys()\n\n        # keys must contain \"row\" or \"rows\"\n        if \"row\" not in keys and \"rows\" not in keys:\n            return _errors(['JSON must have one or other of \"row\" or \"rows\"'])\n        rows = []\n        if \"row\" in keys:\n            if \"rows\" in keys:\n                return _errors(['Cannot use \"row\" and \"rows\" at the same time'])\n            row = data[\"row\"]\n            if not isinstance(row, dict):\n                return _errors(['\"row\" must be a dictionary'])\n            rows = [row]\n            data[\"return\"] = True\n        else:\n            rows = data[\"rows\"]\n        if not isinstance(rows, list):\n            return _errors(['\"rows\" must be a list'])\n        for row in rows:\n            if not isinstance(row, dict):\n                return _errors(['\"rows\" must be a list of dictionaries'])\n\n        # Does this exceed max_insert_rows?\n        max_insert_rows = self.ds.setting(\"max_insert_rows\")\n        if len(rows) > max_insert_rows:\n            return _errors(\n                [\"Too many rows, maximum allowed is {}\".format(max_insert_rows)]\n            )\n\n        # Validate other parameters\n        extras = {\n            key: value for key, value in data.items() if key not in (\"row\", \"rows\")\n        }\n        valid_extras = {\"return\", \"ignore\", \"replace\", \"alter\"}\n        invalid_extras = extras.keys() - valid_extras\n        if invalid_extras:\n            return _errors(\n                ['Invalid parameter: \"{}\"'.format('\", \"'.join(sorted(invalid_extras)))]\n            )\n        if extras.get(\"ignore\") and extras.get(\"replace\"):\n            return _errors(['Cannot use \"ignore\" and \"replace\" at the same time'])\n\n        columns = set(await db.table_columns(table_name))\n        columns.update(pks_list)\n\n        for i, row in enumerate(rows):\n            if upsert:\n                # It MUST have the primary key\n                missing_pks = [pk for pk in pks_list if pk not in row]\n                if missing_pks:\n                    errors.append(\n                        'Row {} is missing primary key column(s): \"{}\"'.format(\n                            i, '\", \"'.join(missing_pks)\n                        )\n                    )\n            invalid_columns = set(row.keys()) - columns\n            if invalid_columns and not extras.get(\"alter\"):\n                errors.append(\n                    \"Row {} has invalid columns: {}\".format(\n                        i, \", \".join(sorted(invalid_columns))\n                    )\n                )\n        if errors:\n            return _errors(errors)\n        return rows, errors, extras\n\n    async def post(self, request, upsert=False):\n        try:\n            resolved = await self.ds.resolve_table(request)\n        except NotFound as e:\n            return _error([e.args[0]], 404)\n        db = resolved.db\n        database_name = db.name\n        table_name = resolved.table\n\n        # Table must exist (may handle table creation in the future)\n        db = self.ds.get_database(database_name)\n        if not await db.table_exists(table_name):\n            return _error([\"Table not found: {}\".format(table_name)], 404)\n\n        if upsert:\n            # Must have insert-row AND upsert-row permissions\n            if not (\n                await self.ds.allowed(\n                    action=\"insert-row\",\n                    resource=TableResource(database=database_name, table=table_name),\n                    actor=request.actor,\n                )\n                and await self.ds.allowed(\n                    action=\"update-row\",\n                    resource=TableResource(database=database_name, table=table_name),\n                    actor=request.actor,\n                )\n            ):\n                return _error(\n                    [\"Permission denied: need both insert-row and update-row\"], 403\n                )\n        else:\n            # Must have insert-row permission\n            if not await self.ds.allowed(\n                action=\"insert-row\",\n                resource=TableResource(database=database_name, table=table_name),\n                actor=request.actor,\n            ):\n                return _error([\"Permission denied\"], 403)\n\n        if not db.is_mutable:\n            return _error([\"Database is immutable\"], 403)\n\n        pks = await db.primary_keys(table_name)\n\n        rows, errors, extras = await self._validate_data(\n            request, db, table_name, pks, upsert\n        )\n        if errors:\n            return _error(errors, 400)\n\n        # Validate column types\n        ct_errors = await _validate_column_types(\n            self.ds, database_name, table_name, rows\n        )\n        if ct_errors:\n            return _error(ct_errors, 400)\n\n        num_rows = len(rows)\n\n        # No that we've passed pks to _validate_data it's safe to\n        # fix the rowids case:\n        if not pks:\n            pks = [\"rowid\"]\n\n        ignore = extras.get(\"ignore\")\n        replace = extras.get(\"replace\")\n        alter = extras.get(\"alter\")\n\n        if upsert and (ignore or replace):\n            return _error([\"Upsert does not support ignore or replace\"], 400)\n\n        if replace and not await self.ds.allowed(\n            action=\"update-row\",\n            resource=TableResource(database=database_name, table=table_name),\n            actor=request.actor,\n        ):\n            return _error(['Permission denied: need update-row to use \"replace\"'], 403)\n\n        initial_schema = None\n        if alter:\n            # Must have alter-table permission\n            if not await self.ds.allowed(\n                action=\"alter-table\",\n                resource=TableResource(database=database_name, table=table_name),\n                actor=request.actor,\n            ):\n                return _error([\"Permission denied for alter-table\"], 403)\n            # Track initial schema to check if it changed later\n            initial_schema = await db.execute_fn(\n                lambda conn: sqlite_utils.Database(conn)[table_name].schema\n            )\n\n        should_return = bool(extras.get(\"return\", False))\n        row_pk_values_for_later = []\n        if should_return and upsert:\n            row_pk_values_for_later = [tuple(row[pk] for pk in pks) for row in rows]\n\n        def insert_or_upsert_rows(conn):\n            table = sqlite_utils.Database(conn)[table_name]\n            kwargs = {}\n            if upsert:\n                kwargs = {\n                    \"pk\": pks[0] if len(pks) == 1 else pks,\n                    \"alter\": alter,\n                }\n            else:\n                # Insert\n                kwargs = {\"ignore\": ignore, \"replace\": replace, \"alter\": alter}\n            if should_return and not upsert:\n                rowids = []\n                method = table.upsert if upsert else table.insert\n                for row in rows:\n                    rowids.append(method(row, **kwargs).last_rowid)\n                return list(\n                    table.rows_where(\n                        \"rowid in ({})\".format(\",\".join(\"?\" for _ in rowids)),\n                        rowids,\n                    )\n                )\n            else:\n                method_all = table.upsert_all if upsert else table.insert_all\n                method_all(rows, **kwargs)\n\n        try:\n            rows = await db.execute_write_fn(insert_or_upsert_rows, request=request)\n        except Exception as e:\n            return _error([str(e)])\n        result = {\"ok\": True}\n        if should_return:\n            if upsert:\n                # Fetch based on initial input IDs\n                where_clause = \" OR \".join(\n                    [\"({})\".format(\" AND \".join(\"{} = ?\".format(pk) for pk in pks))]\n                    * len(row_pk_values_for_later)\n                )\n                args = list(itertools.chain.from_iterable(row_pk_values_for_later))\n                fetched_rows = await db.execute(\n                    \"select {}* from [{}] where {}\".format(\n                        \"rowid, \" if pks == [\"rowid\"] else \"\", table_name, where_clause\n                    ),\n                    args,\n                )\n                result[\"rows\"] = fetched_rows.dicts()\n            else:\n                result[\"rows\"] = rows\n        # We track the number of rows requested, but do not attempt to show which were actually\n        # inserted or upserted v.s. ignored\n        if upsert:\n            await self.ds.track_event(\n                UpsertRowsEvent(\n                    actor=request.actor,\n                    database=database_name,\n                    table=table_name,\n                    num_rows=num_rows,\n                )\n            )\n        else:\n            await self.ds.track_event(\n                InsertRowsEvent(\n                    actor=request.actor,\n                    database=database_name,\n                    table=table_name,\n                    num_rows=num_rows,\n                    ignore=bool(ignore),\n                    replace=bool(replace),\n                )\n            )\n\n        if initial_schema is not None:\n            after_schema = await db.execute_fn(\n                lambda conn: sqlite_utils.Database(conn)[table_name].schema\n            )\n            if initial_schema != after_schema:\n                await self.ds.track_event(\n                    AlterTableEvent(\n                        request.actor,\n                        database=database_name,\n                        table=table_name,\n                        before_schema=initial_schema,\n                        after_schema=after_schema,\n                    )\n                )\n\n        return Response.json(result, status=200 if upsert else 201)\n\n\nclass TableUpsertView(TableInsertView):\n    name = \"table-upsert\"\n\n    async def post(self, request):\n        return await super().post(request, upsert=True)\n\n\nclass TableSetColumnTypeView(BaseView):\n    name = \"table-set-column-type\"\n\n    def __init__(self, datasette):\n        self.ds = datasette\n\n    async def post(self, request):\n        try:\n            resolved = await self.ds.resolve_table(request)\n        except NotFound as e:\n            return _error([e.args[0]], 404)\n\n        database_name = resolved.db.name\n        table_name = resolved.table\n\n        if not await self.ds.allowed(\n            action=\"set-column-type\",\n            resource=TableResource(database=database_name, table=table_name),\n            actor=request.actor,\n        ):\n            return _error([\"Permission denied\"], 403)\n\n        content_type = request.headers.get(\"content-type\") or \"\"\n        if not content_type.startswith(\"application/json\"):\n            return _error([\"Invalid content-type, must be application/json\"], 400)\n\n        try:\n            data = json.loads(await request.post_body())\n        except json.JSONDecodeError as e:\n            return _error([\"Invalid JSON: {}\".format(e)], 400)\n\n        if not isinstance(data, dict):\n            return _error([\"JSON must be a dictionary\"], 400)\n\n        invalid_keys = set(data.keys()) - {\"column\", \"column_type\"}\n        if invalid_keys:\n            return _error(\n                ['Invalid parameter: \"{}\"'.format('\", \"'.join(sorted(invalid_keys)))],\n                400,\n            )\n\n        if \"column\" not in data:\n            return _error(['\"column\" is required'], 400)\n        column = data[\"column\"]\n        if not isinstance(column, str):\n            return _error(['\"column\" must be a string'], 400)\n\n        if \"column_type\" not in data:\n            return _error(['\"column_type\" is required'], 400)\n\n        column_details = await self.ds._get_resource_column_details(\n            database_name, table_name\n        )\n        if column not in column_details:\n            return _error([\"Column not found: {}\".format(column)], 400)\n\n        column_type_data = data[\"column_type\"]\n        if column_type_data is None:\n            await self.ds.remove_column_type(database_name, table_name, column)\n            return Response.json(\n                {\n                    \"ok\": True,\n                    \"database\": database_name,\n                    \"table\": table_name,\n                    \"column\": column,\n                    \"column_type\": None,\n                },\n                status=200,\n            )\n\n        if not isinstance(column_type_data, dict):\n            return _error(['\"column_type\" must be an object or null'], 400)\n\n        invalid_column_type_keys = set(column_type_data.keys()) - {\"type\", \"config\"}\n        if invalid_column_type_keys:\n            return _error(\n                [\n                    'Invalid column_type parameter: \"{}\"'.format(\n                        '\", \"'.join(sorted(invalid_column_type_keys))\n                    )\n                ],\n                400,\n            )\n\n        if \"type\" not in column_type_data:\n            return _error(['\"column_type.type\" is required'], 400)\n        column_type = column_type_data[\"type\"]\n        if not isinstance(column_type, str):\n            return _error(['\"column_type.type\" must be a string'], 400)\n\n        config = column_type_data.get(\"config\")\n        if config is not None and not isinstance(config, dict):\n            return _error(['\"column_type.config\" must be a dictionary'], 400)\n\n        if column_type not in self.ds._column_types:\n            return _error([\"Unknown column type: {}\".format(column_type)], 400)\n\n        try:\n            await self.ds.set_column_type(\n                database_name, table_name, column, column_type, config\n            )\n        except ValueError as e:\n            return _error([str(e)], 400)\n\n        return Response.json(\n            {\n                \"ok\": True,\n                \"database\": database_name,\n                \"table\": table_name,\n                \"column\": column,\n                \"column_type\": {\"type\": column_type, \"config\": config},\n            },\n            status=200,\n        )\n\n\nclass TableDropView(BaseView):\n    name = \"table-drop\"\n\n    def __init__(self, datasette):\n        self.ds = datasette\n\n    async def post(self, request):\n        try:\n            resolved = await self.ds.resolve_table(request)\n        except NotFound as e:\n            return _error([e.args[0]], 404)\n        db = resolved.db\n        database_name = db.name\n        table_name = resolved.table\n        # Table must exist\n        db = self.ds.get_database(database_name)\n        if not await db.table_exists(table_name):\n            return _error([\"Table not found: {}\".format(table_name)], 404)\n        if not await self.ds.allowed(\n            action=\"drop-table\",\n            resource=TableResource(database=database_name, table=table_name),\n            actor=request.actor,\n        ):\n            return _error([\"Permission denied\"], 403)\n        if not db.is_mutable:\n            return _error([\"Database is immutable\"], 403)\n        confirm = False\n        try:\n            data = json.loads(await request.post_body())\n            confirm = data.get(\"confirm\")\n        except json.JSONDecodeError:\n            pass\n\n        if not confirm:\n            return Response.json(\n                {\n                    \"ok\": True,\n                    \"database\": database_name,\n                    \"table\": table_name,\n                    \"row_count\": (\n                        await db.execute(\"select count(*) from [{}]\".format(table_name))\n                    ).single_value(),\n                    \"message\": 'Pass \"confirm\": true to confirm',\n                },\n                status=200,\n            )\n\n        # Drop table\n        def drop_table(conn):\n            sqlite_utils.Database(conn)[table_name].drop()\n\n        await db.execute_write_fn(drop_table, request=request)\n        await self.ds.track_event(\n            DropTableEvent(\n                actor=request.actor, database=database_name, table=table_name\n            )\n        )\n        return Response.json({\"ok\": True}, status=200)\n\n\ndef _get_extras(request):\n    extra_bits = request.args.getlist(\"_extra\")\n    extras = set()\n    for bit in extra_bits:\n        extras.update(bit.split(\",\"))\n    return extras\n\n\nasync def _columns_to_select(table_columns, pks, request):\n    columns = list(table_columns)\n    if \"_col\" in request.args:\n        columns = list(pks)\n        _cols = request.args.getlist(\"_col\")\n        bad_columns = [column for column in _cols if column not in table_columns]\n        if bad_columns:\n            raise DatasetteError(\n                \"_col={} - invalid columns\".format(\", \".join(bad_columns)),\n                status=400,\n            )\n        # De-duplicate maintaining order:\n        columns.extend(dict.fromkeys(_cols))\n    if \"_nocol\" in request.args:\n        # Return all columns EXCEPT these\n        bad_columns = [\n            column\n            for column in request.args.getlist(\"_nocol\")\n            if (column not in table_columns) or (column in pks)\n        ]\n        if bad_columns:\n            raise DatasetteError(\n                \"_nocol={} - invalid columns\".format(\", \".join(bad_columns)),\n                status=400,\n            )\n        tmp_columns = [\n            column for column in columns if column not in request.args.getlist(\"_nocol\")\n        ]\n        columns = tmp_columns\n    return columns\n\n\nasync def _sortable_columns_for_table(datasette, database_name, table_name, use_rowid):\n    db = datasette.databases[database_name]\n    table_metadata = await datasette.table_config(database_name, table_name)\n    if \"sortable_columns\" in table_metadata:\n        sortable_columns = set(table_metadata[\"sortable_columns\"])\n    else:\n        sortable_columns = set(await db.table_columns(table_name))\n    if use_rowid:\n        sortable_columns.add(\"rowid\")\n    return sortable_columns\n\n\nasync def _sort_order(table_metadata, sortable_columns, request, order_by):\n    sort = request.args.get(\"_sort\")\n    sort_desc = request.args.get(\"_sort_desc\")\n\n    if not sort and not sort_desc:\n        sort = table_metadata.get(\"sort\")\n        sort_desc = table_metadata.get(\"sort_desc\")\n\n    if sort and sort_desc:\n        raise DatasetteError(\n            \"Cannot use _sort and _sort_desc at the same time\", status=400\n        )\n\n    if sort:\n        if sort not in sortable_columns:\n            raise DatasetteError(f\"Cannot sort table by {sort}\", status=400)\n\n        order_by = escape_sqlite(sort)\n\n    if sort_desc:\n        if sort_desc not in sortable_columns:\n            raise DatasetteError(f\"Cannot sort table by {sort_desc}\", status=400)\n\n        order_by = f\"{escape_sqlite(sort_desc)} desc\"\n\n    return sort, sort_desc, order_by\n\n\nasync def table_view(datasette, request):\n    await datasette.refresh_schemas()\n    with tracer.trace_child_tasks():\n        response = await table_view_traced(datasette, request)\n\n    # CORS\n    if datasette.cors:\n        add_cors_headers(response.headers)\n\n    # Cache TTL header\n    ttl = request.args.get(\"_ttl\", None)\n    if ttl is None or not ttl.isdigit():\n        ttl = datasette.setting(\"default_cache_ttl\")\n\n    if datasette.cache_headers and response.status == 200:\n        ttl = int(ttl)\n        if ttl == 0:\n            ttl_header = \"no-cache\"\n        else:\n            ttl_header = f\"max-age={ttl}\"\n        response.headers[\"Cache-Control\"] = ttl_header\n\n    # Referrer policy\n    response.headers[\"Referrer-Policy\"] = \"no-referrer\"\n\n    return response\n\n\nasync def table_view_traced(datasette, request):\n    from datasette.app import TableNotFound\n\n    try:\n        resolved = await datasette.resolve_table(request)\n    except TableNotFound as not_found:\n        # Was this actually a canned query?\n        canned_query = await datasette.get_canned_query(\n            not_found.database_name, not_found.table, request.actor\n        )\n        # If this is a canned query, not a table, then dispatch to QueryView instead\n        if canned_query:\n            return await QueryView()(request, datasette)\n        else:\n            raise\n\n    if request.method == \"POST\":\n        return Response.text(\"Method not allowed\", status=405)\n\n    format_ = request.url_vars.get(\"format\") or \"html\"\n    extra_extras = None\n    context_for_html_hack = False\n    default_labels = False\n    if format_ == \"html\":\n        extra_extras = {\"_html\"}\n        context_for_html_hack = True\n        default_labels = True\n\n    view_data = await table_view_data(\n        datasette,\n        request,\n        resolved,\n        extra_extras=extra_extras,\n        context_for_html_hack=context_for_html_hack,\n        default_labels=default_labels,\n    )\n    if isinstance(view_data, Response):\n        return view_data\n    data, rows, columns, expanded_columns, sql, next_url = view_data\n\n    # Handle formats from plugins\n    if format_ == \"csv\":\n\n        async def fetch_data(request, _next=None):\n            (\n                data,\n                rows,\n                columns,\n                expanded_columns,\n                sql,\n                next_url,\n            ) = await table_view_data(\n                datasette,\n                request,\n                resolved,\n                extra_extras=extra_extras,\n                context_for_html_hack=context_for_html_hack,\n                default_labels=default_labels,\n                _next=_next,\n            )\n            data[\"rows\"] = rows\n            data[\"table\"] = resolved.table\n            data[\"columns\"] = columns\n            data[\"expanded_columns\"] = expanded_columns\n            return data, None, None\n\n        return await stream_csv(datasette, fetch_data, request, resolved.db.name)\n    elif format_ in datasette.renderers.keys():\n        # Dispatch request to the correct output format renderer\n        # (CSV is not handled here due to streaming)\n        result = call_with_supported_arguments(\n            datasette.renderers[format_][0],\n            datasette=datasette,\n            columns=columns,\n            rows=rows,\n            sql=sql,\n            query_name=None,\n            database=resolved.db.name,\n            table=resolved.table,\n            request=request,\n            view_name=\"table\",\n            truncated=False,\n            error=None,\n            # These will be deprecated in Datasette 1.0:\n            args=request.args,\n            data=data,\n        )\n        if asyncio.iscoroutine(result):\n            result = await result\n        if result is None:\n            raise NotFound(\"No data\")\n        if isinstance(result, dict):\n            r = Response(\n                body=result.get(\"body\"),\n                status=result.get(\"status_code\") or 200,\n                content_type=result.get(\"content_type\", \"text/plain\"),\n                headers=result.get(\"headers\"),\n            )\n        elif isinstance(result, Response):\n            r = result\n            # if status_code is not None:\n            #     # Over-ride the status code\n            #     r.status = status_code\n        else:\n            assert False, f\"{result} should be dict or Response\"\n    elif format_ == \"html\":\n        headers = {}\n        templates = [\n            f\"table-{to_css_class(resolved.db.name)}-{to_css_class(resolved.table)}.html\",\n            \"table.html\",\n        ]\n        environment = datasette.get_jinja_environment(request)\n        template = environment.select_template(templates)\n        alternate_url_json = datasette.absolute_url(\n            request,\n            datasette.urls.path(path_with_format(request=request, format=\"json\")),\n        )\n        headers.update(\n            {\n                \"Link\": '<{}>; rel=\"alternate\"; type=\"application/json+datasette\"'.format(\n                    alternate_url_json\n                )\n            }\n        )\n        r = Response.html(\n            await datasette.render_template(\n                template,\n                dict(\n                    data,\n                    append_querystring=append_querystring,\n                    path_with_replaced_args=path_with_replaced_args,\n                    fix_path=datasette.urls.path,\n                    settings=datasette.settings_dict(),\n                    # TODO: review up all of these hacks:\n                    alternate_url_json=alternate_url_json,\n                    datasette_allow_facet=(\n                        \"true\" if datasette.setting(\"allow_facet\") else \"false\"\n                    ),\n                    is_sortable=any(c[\"sortable\"] for c in data[\"display_columns\"]),\n                    allow_execute_sql=await datasette.allowed(\n                        action=\"execute-sql\",\n                        resource=DatabaseResource(database=resolved.db.name),\n                        actor=request.actor,\n                    ),\n                    query_ms=1.2,\n                    select_templates=[\n                        f\"{'*' if template_name == template.name else ''}{template_name}\"\n                        for template_name in templates\n                    ],\n                    top_table=make_slot_function(\n                        \"top_table\",\n                        datasette,\n                        request,\n                        database=resolved.db.name,\n                        table=resolved.table,\n                    ),\n                    count_limit=resolved.db.count_limit,\n                ),\n                request=request,\n                view_name=\"table\",\n            ),\n            headers=headers,\n        )\n    else:\n        assert False, \"Invalid format: {}\".format(format_)\n    if next_url:\n        r.headers[\"link\"] = f'<{next_url}>; rel=\"next\"'\n    return r\n\n\nasync def table_view_data(\n    datasette,\n    request,\n    resolved,\n    extra_extras=None,\n    context_for_html_hack=False,\n    default_labels=False,\n    _next=None,\n):\n    extra_extras = extra_extras or set()\n    # We have a table or view\n    db = resolved.db\n    database_name = resolved.db.name\n    table_name = resolved.table\n    is_view = resolved.is_view\n\n    # Can this user view it?\n    visible, private = await datasette.check_visibility(\n        request.actor,\n        action=\"view-table\",\n        resource=TableResource(database=database_name, table=table_name),\n    )\n    if not visible:\n        raise Forbidden(\"You do not have permission to view this table\")\n\n    # Redirect based on request.args, if necessary\n    redirect_response = await _redirect_if_needed(datasette, request, resolved)\n    if redirect_response:\n        return redirect_response\n\n    # Introspect columns and primary keys for table\n    pks = await db.primary_keys(table_name)\n    table_columns = await db.table_columns(table_name)\n\n    # Take ?_col= and ?_nocol= into account\n    specified_columns = await _columns_to_select(table_columns, pks, request)\n    select_specified_columns = \", \".join(escape_sqlite(t) for t in specified_columns)\n    select_all_columns = \", \".join(escape_sqlite(t) for t in table_columns)\n\n    # rowid tables (no specified primary key) need a different SELECT\n    use_rowid = not pks and not is_view\n    order_by = \"\"\n    if use_rowid:\n        select_specified_columns = f\"rowid, {select_specified_columns}\"\n        select_all_columns = f\"rowid, {select_all_columns}\"\n        order_by = \"rowid\"\n        order_by_pks = \"rowid\"\n    else:\n        order_by_pks = \", \".join([escape_sqlite(pk) for pk in pks])\n        order_by = order_by_pks\n\n    if is_view:\n        order_by = \"\"\n\n    # TODO: This logic should turn into logic about which ?_extras get\n    # executed instead:\n    nocount = request.args.get(\"_nocount\")\n    nofacet = request.args.get(\"_nofacet\")\n    nosuggest = request.args.get(\"_nosuggest\")\n    if request.args.get(\"_shape\") in (\"array\", \"object\"):\n        nocount = True\n        nofacet = True\n\n    table_metadata = await datasette.table_config(database_name, table_name)\n\n    # Arguments that start with _ and don't contain a __ are\n    # special - things like ?_search= - and should not be\n    # treated as filters.\n    filter_args = []\n    for key in request.args:\n        if not (key.startswith(\"_\") and \"__\" not in key):\n            for v in request.args.getlist(key):\n                filter_args.append((key, v))\n\n    # Build where clauses from query string arguments\n    filters = Filters(sorted(filter_args))\n    where_clauses, params = filters.build_where_clauses(table_name)\n\n    # Execute filters_from_request plugin hooks - including the default\n    # ones that live in datasette/filters.py\n    extra_context_from_filters = {}\n    extra_human_descriptions = []\n\n    for hook in pm.hook.filters_from_request(\n        request=request,\n        table=table_name,\n        database=database_name,\n        datasette=datasette,\n    ):\n        filter_arguments = await await_me_maybe(hook)\n        if filter_arguments:\n            where_clauses.extend(filter_arguments.where_clauses)\n            params.update(filter_arguments.params)\n            extra_human_descriptions.extend(filter_arguments.human_descriptions)\n            extra_context_from_filters.update(filter_arguments.extra_context)\n\n    # Deal with custom sort orders\n    sortable_columns = await _sortable_columns_for_table(\n        datasette, database_name, table_name, use_rowid\n    )\n\n    sort, sort_desc, order_by = await _sort_order(\n        table_metadata, sortable_columns, request, order_by\n    )\n\n    from_sql = \"from {table_name} {where}\".format(\n        table_name=escape_sqlite(table_name),\n        where=(\n            (\"where {} \".format(\" and \".join(where_clauses))) if where_clauses else \"\"\n        ),\n    )\n    # Copy of params so we can mutate them later:\n    from_sql_params = dict(**params)\n\n    count_sql = f\"select count(*) {from_sql}\"\n\n    # Handle pagination driven by ?_next=\n    _next = _next or request.args.get(\"_next\")\n\n    offset = \"\"\n    if _next:\n        sort_value = None\n        if is_view:\n            # _next is an offset\n            offset = f\" offset {int(_next)}\"\n        else:\n            components = urlsafe_components(_next)\n            # If a sort order is applied and there are multiple components,\n            # the first of these is the sort value\n            if (sort or sort_desc) and (len(components) > 1):\n                sort_value = components[0]\n                # Special case for if non-urlencoded first token was $null\n                if _next.split(\",\")[0] == \"$null\":\n                    sort_value = None\n                components = components[1:]\n\n            # Figure out the SQL for next-based-on-primary-key first\n            next_by_pk_clauses = []\n            if use_rowid:\n                next_by_pk_clauses.append(f\"rowid > :p{len(params)}\")\n                params[f\"p{len(params)}\"] = components[0]\n            else:\n                # Apply the tie-breaker based on primary keys\n                if len(components) == len(pks):\n                    param_len = len(params)\n                    next_by_pk_clauses.append(compound_keys_after_sql(pks, param_len))\n                    for i, pk_value in enumerate(components):\n                        params[f\"p{param_len + i}\"] = pk_value\n\n            # Now add the sort SQL, which may incorporate next_by_pk_clauses\n            if sort or sort_desc:\n                if sort_value is None:\n                    if sort_desc:\n                        # Just items where column is null ordered by pk\n                        where_clauses.append(\n                            \"({column} is null and {next_clauses})\".format(\n                                column=escape_sqlite(sort_desc),\n                                next_clauses=\" and \".join(next_by_pk_clauses),\n                            )\n                        )\n                    else:\n                        where_clauses.append(\n                            \"({column} is not null or ({column} is null and {next_clauses}))\".format(\n                                column=escape_sqlite(sort),\n                                next_clauses=\" and \".join(next_by_pk_clauses),\n                            )\n                        )\n                else:\n                    where_clauses.append(\n                        \"({column} {op} :p{p}{extra_desc_only} or ({column} = :p{p} and {next_clauses}))\".format(\n                            column=escape_sqlite(sort or sort_desc),\n                            op=\">\" if sort else \"<\",\n                            p=len(params),\n                            extra_desc_only=(\n                                \"\"\n                                if sort\n                                else \" or {column2} is null\".format(\n                                    column2=escape_sqlite(sort or sort_desc)\n                                )\n                            ),\n                            next_clauses=\" and \".join(next_by_pk_clauses),\n                        )\n                    )\n                    params[f\"p{len(params)}\"] = sort_value\n                order_by = f\"{order_by}, {order_by_pks}\"\n            else:\n                where_clauses.extend(next_by_pk_clauses)\n\n    where_clause = \"\"\n    if where_clauses:\n        where_clause = f\"where {' and '.join(where_clauses)} \"\n\n    if order_by:\n        order_by = f\"order by {order_by}\"\n\n    extra_args = {}\n    # Handle ?_size=500\n    # TODO: This was:\n    # page_size = _size or request.args.get(\"_size\") or table_metadata.get(\"size\")\n    page_size = request.args.get(\"_size\") or table_metadata.get(\"size\")\n    if page_size:\n        if page_size == \"max\":\n            page_size = datasette.max_returned_rows\n        try:\n            page_size = int(page_size)\n            if page_size < 0:\n                raise ValueError\n\n        except ValueError:\n            raise BadRequest(\"_size must be a positive integer\")\n\n        if page_size > datasette.max_returned_rows:\n            raise BadRequest(f\"_size must be <= {datasette.max_returned_rows}\")\n\n        extra_args[\"page_size\"] = page_size\n    else:\n        page_size = datasette.page_size\n\n    # Facets are calculated against SQL without order by or limit\n    sql_no_order_no_limit = (\n        \"select {select_all_columns} from {table_name} {where}\".format(\n            select_all_columns=select_all_columns,\n            table_name=escape_sqlite(table_name),\n            where=where_clause,\n        )\n    )\n\n    # This is the SQL that populates the main table on the page\n    sql = \"select {select_specified_columns} from {table_name} {where}{order_by} limit {page_size}{offset}\".format(\n        select_specified_columns=select_specified_columns,\n        table_name=escape_sqlite(table_name),\n        where=where_clause,\n        order_by=order_by,\n        page_size=page_size + 1,\n        offset=offset,\n    )\n\n    if request.args.get(\"_timelimit\"):\n        extra_args[\"custom_time_limit\"] = int(request.args.get(\"_timelimit\"))\n\n    # Execute the main query!\n    try:\n        results = await db.execute(sql, params, truncate=True, **extra_args)\n    except (sqlite3.OperationalError, InvalidSql) as e:\n        raise DatasetteError(str(e), title=\"Invalid SQL\", status=400)\n\n    except sqlite3.OperationalError as e:\n        raise DatasetteError(str(e))\n\n    columns = [r[0] for r in results.description]\n    rows = list(results.rows)\n\n    # Expand labeled columns if requested\n    expanded_columns = []\n    # List of (fk_dict, label_column-or-None) pairs for that table\n    expandable_columns = []\n    for fk in await db.foreign_keys_for_table(table_name):\n        label_column = await db.label_column_for_table(fk[\"other_table\"])\n        expandable_columns.append((fk, label_column))\n\n    columns_to_expand = None\n    try:\n        all_labels = value_as_boolean(request.args.get(\"_labels\", \"\"))\n    except ValueError:\n        all_labels = default_labels\n    # Check for explicit _label=\n    if \"_label\" in request.args:\n        columns_to_expand = request.args.getlist(\"_label\")\n    if columns_to_expand is None and all_labels:\n        # expand all columns with foreign keys\n        columns_to_expand = [fk[\"column\"] for fk, _ in expandable_columns]\n\n    if columns_to_expand:\n        expanded_labels = {}\n        for fk, _ in expandable_columns:\n            column = fk[\"column\"]\n            if column not in columns_to_expand:\n                continue\n            if column not in columns:\n                continue\n            expanded_columns.append(column)\n            # Gather the values\n            column_index = columns.index(column)\n            values = [row[column_index] for row in rows]\n            # Expand them\n            expanded_labels.update(\n                await datasette.expand_foreign_keys(\n                    request.actor, database_name, table_name, column, values\n                )\n            )\n        if expanded_labels:\n            # Rewrite the rows\n            new_rows = []\n            for row in rows:\n                new_row = CustomRow(columns)\n                for column in row.keys():\n                    value = row[column]\n                    if (column, value) in expanded_labels and value is not None:\n                        new_row[column] = {\n                            \"value\": value,\n                            \"label\": expanded_labels[(column, value)],\n                        }\n                    else:\n                        new_row[column] = value\n                new_rows.append(new_row)\n            rows = new_rows\n\n    _next = request.args.get(\"_next\")\n\n    # Pagination next link\n    next_value, next_url = await _next_value_and_url(\n        datasette,\n        db,\n        request,\n        table_name,\n        _next,\n        rows,\n        pks,\n        use_rowid,\n        sort,\n        sort_desc,\n        page_size,\n        is_view,\n    )\n    rows = rows[:page_size]\n\n    # Resolve extras\n    extras = _get_extras(request)\n    if any(k for k in request.args.keys() if k == \"_facet\" or k.startswith(\"_facet_\")):\n        extras.add(\"facet_results\")\n    if request.args.get(\"_shape\") == \"object\":\n        extras.add(\"primary_keys\")\n    if extra_extras:\n        extras.update(extra_extras)\n\n    async def extra_count_sql():\n        return count_sql\n\n    async def extra_count():\n        \"Total count of rows matching these filters\"\n        # Calculate the total count for this query\n        count = None\n        if (\n            not db.is_mutable\n            and datasette.inspect_data\n            and count_sql == f\"select count(*) from {table_name} \"\n        ):\n            # We can use a previously cached table row count\n            try:\n                count = datasette.inspect_data[database_name][\"tables\"][table_name][\n                    \"count\"\n                ]\n            except KeyError:\n                pass\n\n        # Otherwise run a select count(*) ...\n        if count_sql and count is None and not nocount:\n            count_sql_limited = (\n                f\"select count(*) from (select * {from_sql} limit 10001)\"\n            )\n            try:\n                count_rows = list(await db.execute(count_sql_limited, from_sql_params))\n                count = count_rows[0][0]\n            except QueryInterrupted:\n                pass\n        return count\n\n    async def facet_instances(extra_count):\n        facet_instances = []\n        facet_classes = list(\n            itertools.chain.from_iterable(pm.hook.register_facet_classes())\n        )\n        for facet_class in facet_classes:\n            facet_instances.append(\n                facet_class(\n                    datasette,\n                    request,\n                    database_name,\n                    sql=sql_no_order_no_limit,\n                    params=params,\n                    table=table_name,\n                    table_config=table_metadata,\n                    row_count=extra_count,\n                )\n            )\n        return facet_instances\n\n    async def extra_facet_results(facet_instances):\n        \"Results of facets calculated against this data\"\n        facet_results = {}\n        facets_timed_out = []\n\n        if not nofacet:\n            # Run them in parallel\n            facet_awaitables = [facet.facet_results() for facet in facet_instances]\n            facet_awaitable_results = await run_sequential(*facet_awaitables)\n            for (\n                instance_facet_results,\n                instance_facets_timed_out,\n            ) in facet_awaitable_results:\n                for facet_info in instance_facet_results:\n                    base_key = facet_info[\"name\"]\n                    key = base_key\n                    i = 1\n                    while key in facet_results:\n                        i += 1\n                        key = f\"{base_key}_{i}\"\n                    facet_results[key] = facet_info\n                facets_timed_out.extend(instance_facets_timed_out)\n\n        return {\n            \"results\": facet_results,\n            \"timed_out\": facets_timed_out,\n        }\n\n    async def extra_suggested_facets(facet_instances):\n        \"Suggestions for facets that might return interesting results\"\n        suggested_facets = []\n        # Calculate suggested facets\n        if (\n            datasette.setting(\"suggest_facets\")\n            and datasette.setting(\"allow_facet\")\n            and not _next\n            and not nofacet\n            and not nosuggest\n        ):\n            # Run them in parallel\n            facet_suggest_awaitables = [facet.suggest() for facet in facet_instances]\n            for suggest_result in await run_sequential(*facet_suggest_awaitables):\n                suggested_facets.extend(suggest_result)\n        return suggested_facets\n\n    # Faceting\n    if not datasette.setting(\"allow_facet\") and any(\n        arg.startswith(\"_facet\") for arg in request.args\n    ):\n        raise BadRequest(\"_facet= is not allowed\")\n\n    # human_description_en combines filters AND search, if provided\n    async def extra_human_description_en():\n        \"Human-readable description of the filters\"\n        human_description_en = filters.human_description_en(\n            extra=extra_human_descriptions\n        )\n        if sort or sort_desc:\n            human_description_en = \" \".join(\n                [b for b in [human_description_en, sorted_by] if b]\n            )\n        return human_description_en\n\n    if sort or sort_desc:\n        sorted_by = \"sorted by {}{}\".format(\n            (sort or sort_desc), \" descending\" if sort_desc else \"\"\n        )\n\n    async def extra_next_url():\n        \"Full URL for the next page of results\"\n        return next_url\n\n    async def extra_columns():\n        \"Column names returned by this query\"\n        return columns\n\n    async def extra_all_columns():\n        \"All columns in the table, regardless of _col/_nocol filtering\"\n        return list(table_columns)\n\n    async def extra_primary_keys():\n        \"Primary keys for this table\"\n        return pks\n\n    async def extra_actions():\n        async def actions():\n            links = []\n            kwargs = {\n                \"datasette\": datasette,\n                \"database\": database_name,\n                \"actor\": request.actor,\n                \"request\": request,\n            }\n            if is_view:\n                kwargs[\"view\"] = table_name\n                method = pm.hook.view_actions\n            else:\n                kwargs[\"table\"] = table_name\n                method = pm.hook.table_actions\n            for hook in method(**kwargs):\n                extra_links = await await_me_maybe(hook)\n                if extra_links:\n                    links.extend(extra_links)\n            return links\n\n        return actions\n\n    async def extra_is_view():\n        return is_view\n\n    async def extra_debug():\n        \"Extra debug information\"\n        return {\n            \"resolved\": repr(resolved),\n            \"url_vars\": request.url_vars,\n            \"nofacet\": nofacet,\n            \"nosuggest\": nosuggest,\n        }\n\n    async def extra_request():\n        \"Full information about the request\"\n        return {\n            \"url\": request.url,\n            \"path\": request.path,\n            \"full_path\": request.full_path,\n            \"host\": request.host,\n            \"args\": request.args._data,\n        }\n\n    async def run_display_columns_and_rows():\n        display_columns, display_rows = await display_columns_and_rows(\n            datasette,\n            database_name,\n            table_name,\n            results.description,\n            rows,\n            link_column=not is_view,\n            truncate_cells=datasette.setting(\"truncate_cells_html\"),\n            sortable_columns=sortable_columns,\n            request=request,\n        )\n        return {\n            \"columns\": display_columns,\n            \"rows\": display_rows,\n        }\n\n    async def extra_display_columns(run_display_columns_and_rows):\n        return run_display_columns_and_rows[\"columns\"]\n\n    async def extra_display_rows(run_display_columns_and_rows):\n        return run_display_columns_and_rows[\"rows\"]\n\n    async def extra_render_cell():\n        \"Rendered HTML for each cell using the render_cell plugin hook\"\n        pks_for_display = pks if pks else ([\"rowid\"] if not is_view else [])\n        col_names = [col[0] for col in results.description]\n        ct_map = await datasette.get_column_types(database_name, table_name)\n        rendered_rows = []\n        for row in rows:\n            rendered_row = {}\n            for value, column in zip(row, col_names):\n                ct = ct_map.get(column)\n                plugin_display_value = None\n                # Try column type render_cell first\n                if ct:\n                    candidate = await ct.render_cell(\n                        value=value,\n                        column=column,\n                        table=table_name,\n                        database=database_name,\n                        datasette=datasette,\n                        request=request,\n                    )\n                    if candidate is not None:\n                        plugin_display_value = candidate\n                if plugin_display_value is None:\n                    for candidate in pm.hook.render_cell(\n                        row=row,\n                        value=value,\n                        column=column,\n                        table=table_name,\n                        pks=pks_for_display,\n                        database=database_name,\n                        datasette=datasette,\n                        request=request,\n                        column_type=ct,\n                    ):\n                        candidate = await await_me_maybe(candidate)\n                        if candidate is not None:\n                            plugin_display_value = candidate\n                            break\n                if plugin_display_value:\n                    rendered_row[column] = str(plugin_display_value)\n            rendered_rows.append(rendered_row)\n        return rendered_rows\n\n    async def extra_query():\n        \"Details of the underlying SQL query\"\n        return {\n            \"sql\": sql,\n            \"params\": params,\n        }\n\n    async def extra_column_types():\n        \"Column type assignments for this table\"\n        ct_map = await datasette.get_column_types(database_name, table_name)\n        return {\n            col_name: {\n                \"type\": ct.name,\n                \"config\": ct.config,\n            }\n            for col_name, ct in ct_map.items()\n        }\n\n    async def extra_set_column_type_ui():\n        \"Column type UI metadata for this table\"\n        if is_view:\n            return None\n\n        if not await datasette.allowed(\n            action=\"set-column-type\",\n            resource=TableResource(database=database_name, table=table_name),\n            actor=request.actor,\n        ):\n            return None\n\n        column_details = await datasette._get_resource_column_details(\n            database_name, table_name\n        )\n        ct_map = await datasette.get_column_types(database_name, table_name)\n        columns = {}\n        for column_name, column_detail in column_details.items():\n            current = ct_map.get(column_name)\n            columns[column_name] = {\n                \"current\": (\n                    {\"type\": current.name, \"config\": current.config}\n                    if current is not None\n                    else None\n                ),\n                \"options\": [\n                    {\n                        \"name\": name,\n                        \"description\": ct_cls.description,\n                    }\n                    for name, ct_cls in sorted(datasette._column_types.items())\n                    if datasette._column_type_is_applicable(ct_cls, column_detail)\n                ],\n            }\n        return {\n            \"path\": \"{}/-/set-column-type\".format(\n                datasette.urls.table(database_name, table_name)\n            ),\n            \"columns\": columns,\n        }\n\n    async def extra_metadata():\n        \"Metadata about the table and database\"\n        tablemetadata = await datasette.get_resource_metadata(database_name, table_name)\n\n        rows = await datasette.get_internal_database().execute(\n            \"\"\"\n              SELECT\n                column_name,\n                value\n              FROM metadata_columns\n              WHERE database_name = ?\n                AND resource_name = ?\n                AND key = 'description'\n            \"\"\",\n            [database_name, table_name],\n        )\n        tablemetadata[\"columns\"] = dict(rows)\n        return tablemetadata\n\n    async def extra_database():\n        return database_name\n\n    async def extra_table():\n        return table_name\n\n    async def extra_database_color():\n        return db.color\n\n    async def extra_form_hidden_args():\n        form_hidden_args = []\n        for key in request.args:\n            if (\n                key.startswith(\"_\")\n                and key not in (\"_sort\", \"_sort_desc\", \"_search\", \"_next\")\n                and \"__\" not in key\n            ):\n                for value in request.args.getlist(key):\n                    form_hidden_args.append((key, value))\n        return form_hidden_args\n\n    async def extra_filters():\n        return filters\n\n    async def extra_custom_table_templates():\n        return [\n            f\"_table-{to_css_class(database_name)}-{to_css_class(table_name)}.html\",\n            f\"_table-table-{to_css_class(database_name)}-{to_css_class(table_name)}.html\",\n            \"_table.html\",\n        ]\n\n    async def extra_sorted_facet_results(extra_facet_results):\n        facet_configs = table_metadata.get(\"facets\", [])\n        if facet_configs:\n            # Build ordered list of facet names from metadata config\n            metadata_facet_names = []\n            for fc in facet_configs:\n                if isinstance(fc, str):\n                    metadata_facet_names.append(fc)\n                elif isinstance(fc, dict):\n                    metadata_facet_names.append(list(fc.values())[0])\n            metadata_order = {name: i for i, name in enumerate(metadata_facet_names)}\n            metadata_facets = []\n            request_facets = []\n            for f in extra_facet_results[\"results\"].values():\n                if f[\"name\"] in metadata_order:\n                    metadata_facets.append(f)\n                else:\n                    request_facets.append(f)\n            metadata_facets.sort(key=lambda f: metadata_order[f[\"name\"]])\n            request_facets.sort(\n                key=lambda f: (len(f[\"results\"]), f[\"name\"]),\n                reverse=True,\n            )\n            return metadata_facets + request_facets\n        else:\n            return sorted(\n                extra_facet_results[\"results\"].values(),\n                key=lambda f: (len(f[\"results\"]), f[\"name\"]),\n                reverse=True,\n            )\n\n    async def extra_table_definition():\n        return await db.get_table_definition(table_name)\n\n    async def extra_view_definition():\n        return await db.get_view_definition(table_name)\n\n    async def extra_renderers(extra_expandable_columns, extra_query):\n        renderers = {}\n        url_labels_extra = {}\n        if extra_expandable_columns:\n            url_labels_extra = {\"_labels\": \"on\"}\n        for key, (_, can_render) in datasette.renderers.items():\n            it_can_render = call_with_supported_arguments(\n                can_render,\n                datasette=datasette,\n                columns=columns or [],\n                rows=rows or [],\n                sql=extra_query.get(\"sql\", None),\n                query_name=None,\n                database=database_name,\n                table=table_name,\n                request=request,\n                view_name=\"table\",\n            )\n            it_can_render = await await_me_maybe(it_can_render)\n            if it_can_render:\n                renderers[key] = datasette.urls.path(\n                    path_with_format(\n                        request=request, format=key, extra_qs={**url_labels_extra}\n                    )\n                )\n        return renderers\n\n    async def extra_private():\n        return private\n\n    async def extra_expandable_columns():\n        expandables = []\n        db = datasette.databases[database_name]\n        for fk in await db.foreign_keys_for_table(table_name):\n            label_column = await db.label_column_for_table(fk[\"other_table\"])\n            expandables.append((fk, label_column))\n        return expandables\n\n    async def extra_extras():\n        \"Available ?_extra= blocks\"\n        all_extras = [\n            (key[len(\"extra_\") :], fn.__doc__)\n            for key, fn in registry._registry.items()\n            if key.startswith(\"extra_\")\n        ]\n        return [\n            {\n                \"name\": name,\n                \"description\": doc,\n                \"toggle_url\": datasette.absolute_url(\n                    request,\n                    datasette.urls.path(\n                        path_with_added_args(request, {\"_extra\": name})\n                        if name not in extras\n                        else path_with_removed_args(request, {\"_extra\": name})\n                    ),\n                ),\n                \"selected\": name in extras,\n            }\n            for name, doc in all_extras\n        ]\n\n    async def extra_facets_timed_out(extra_facet_results):\n        return extra_facet_results[\"timed_out\"]\n\n    bundles = {\n        \"html\": [\n            \"suggested_facets\",\n            \"facet_results\",\n            \"facets_timed_out\",\n            \"count\",\n            \"count_sql\",\n            \"human_description_en\",\n            \"next_url\",\n            \"metadata\",\n            \"query\",\n            \"columns\",\n            \"display_columns\",\n            \"display_rows\",\n            \"database\",\n            \"table\",\n            \"database_color\",\n            \"actions\",\n            \"filters\",\n            \"renderers\",\n            \"custom_table_templates\",\n            \"sorted_facet_results\",\n            \"table_definition\",\n            \"view_definition\",\n            \"is_view\",\n            \"private\",\n            \"primary_keys\",\n            \"all_columns\",\n            \"expandable_columns\",\n            \"form_hidden_args\",\n            \"set_column_type_ui\",\n        ]\n    }\n\n    for key, values in bundles.items():\n        if f\"_{key}\" in extras:\n            extras.update(values)\n        extras.discard(f\"_{key}\")\n\n    registry = Registry(\n        extra_count,\n        extra_count_sql,\n        extra_facet_results,\n        extra_facets_timed_out,\n        extra_suggested_facets,\n        facet_instances,\n        extra_human_description_en,\n        extra_next_url,\n        extra_columns,\n        extra_all_columns,\n        extra_primary_keys,\n        run_display_columns_and_rows,\n        extra_display_columns,\n        extra_display_rows,\n        extra_render_cell,\n        extra_debug,\n        extra_request,\n        extra_query,\n        extra_column_types,\n        extra_set_column_type_ui,\n        extra_metadata,\n        extra_extras,\n        extra_database,\n        extra_table,\n        extra_database_color,\n        extra_actions,\n        extra_filters,\n        extra_renderers,\n        extra_custom_table_templates,\n        extra_sorted_facet_results,\n        extra_table_definition,\n        extra_view_definition,\n        extra_is_view,\n        extra_private,\n        extra_expandable_columns,\n        extra_form_hidden_args,\n    )\n\n    results = await registry.resolve_multi(\n        [\"extra_{}\".format(extra) for extra in extras]\n    )\n    data = {\n        \"ok\": True,\n        \"next\": next_value and str(next_value) or None,\n    }\n    data.update(\n        {\n            key.replace(\"extra_\", \"\"): value\n            for key, value in results.items()\n            if key.startswith(\"extra_\") and key.replace(\"extra_\", \"\") in extras\n        }\n    )\n    raw_sqlite_rows = rows[:page_size]\n    # Apply transform_value for columns with assigned types\n    ct_map = await datasette.get_column_types(database_name, table_name)\n    transformed_rows = []\n    for r in raw_sqlite_rows:\n        row_dict = dict(r)\n        for col_name, ct in ct_map.items():\n            if col_name in row_dict:\n                row_dict[col_name] = await ct.transform_value(\n                    row_dict[col_name], datasette\n                )\n        transformed_rows.append(row_dict)\n    data[\"rows\"] = transformed_rows\n\n    if context_for_html_hack:\n        data.update(extra_context_from_filters)\n        # filter_columns combine the columns we know are available\n        # in the table with any additional columns (such as rowid)\n        # which are available in the query\n        data[\"filter_columns\"] = list(columns) + [\n            table_column\n            for table_column in table_columns\n            if table_column not in columns\n        ]\n        url_labels_extra = {}\n        if data.get(\"expandable_columns\"):\n            url_labels_extra = {\"_labels\": \"on\"}\n        url_csv_args = {\"_size\": \"max\", **url_labels_extra}\n        url_csv = datasette.urls.path(\n            path_with_format(request=request, format=\"csv\", extra_qs=url_csv_args)\n        )\n        url_csv_path = url_csv.split(\"?\")[0]\n        data.update(\n            {\n                \"url_csv\": url_csv,\n                \"url_csv_path\": url_csv_path,\n                \"url_csv_hidden_args\": [\n                    (key, value)\n                    for key, value in urllib.parse.parse_qsl(request.query_string)\n                    if key not in (\"_labels\", \"_facet\", \"_size\")\n                ]\n                + [(\"_size\", \"max\")],\n            }\n        )\n        # if no sort specified AND table has a single primary key,\n        # set sort to that so arrow is displayed\n        if not sort and not sort_desc:\n            if 1 == len(pks):\n                sort = pks[0]\n            elif use_rowid:\n                sort = \"rowid\"\n        data[\"sort\"] = sort\n        data[\"sort_desc\"] = sort_desc\n\n    return data, rows[:page_size], columns, expanded_columns, sql, next_url\n\n\nasync def _next_value_and_url(\n    datasette,\n    db,\n    request,\n    table_name,\n    _next,\n    rows,\n    pks,\n    use_rowid,\n    sort,\n    sort_desc,\n    page_size,\n    is_view,\n):\n    next_value = None\n    next_url = None\n    if 0 < page_size < len(rows):\n        if is_view:\n            next_value = int(_next or 0) + page_size\n        else:\n            next_value = path_from_row_pks(rows[-2], pks, use_rowid)\n        # If there's a sort or sort_desc, add that value as a prefix\n        if (sort or sort_desc) and not is_view:\n            try:\n                prefix = rows[-2][sort or sort_desc]\n            except IndexError:\n                # sort/sort_desc column missing from SELECT - look up value by PK instead\n                prefix_where_clause = \" and \".join(\n                    \"[{}] = :pk{}\".format(pk, i) for i, pk in enumerate(pks)\n                )\n                prefix_lookup_sql = \"select [{}] from [{}] where {}\".format(\n                    sort or sort_desc, table_name, prefix_where_clause\n                )\n                prefix = (\n                    await db.execute(\n                        prefix_lookup_sql,\n                        {\n                            **{\n                                \"pk{}\".format(i): rows[-2][pk]\n                                for i, pk in enumerate(pks)\n                            }\n                        },\n                    )\n                ).single_value()\n            if isinstance(prefix, dict) and \"value\" in prefix:\n                prefix = prefix[\"value\"]\n            if prefix is None:\n                prefix = \"$null\"\n            else:\n                prefix = tilde_encode(str(prefix))\n            next_value = f\"{prefix},{next_value}\"\n            added_args = {\"_next\": next_value}\n            if sort:\n                added_args[\"_sort\"] = sort\n            else:\n                added_args[\"_sort_desc\"] = sort_desc\n        else:\n            added_args = {\"_next\": next_value}\n        next_url = datasette.absolute_url(\n            request, datasette.urls.path(path_with_replaced_args(request, added_args))\n        )\n    return next_value, next_url\n"
  },
  {
    "path": "demos/apache-proxy/000-default.conf",
    "content": "\n    Options Indexes FollowSymLinks\n    AllowOverride None\n    Require all granted\n\n\n\n    ServerName localhost\n    DocumentRoot /app/html\n    ProxyPreserveHost On\n    ProxyPass /prefix/ http://127.0.0.1:8001/\n    Header add X-Proxied-By \"Apache2 Debian\"\n\n"
  },
  {
    "path": "demos/apache-proxy/Dockerfile",
    "content": "FROM python:3.11.0-slim-bullseye\n\nRUN apt-get update && \\\n    apt-get install -y apache2 supervisor && \\\n    apt clean && \\\n    rm -rf /var/lib/apt && \\\n    rm -rf /var/lib/dpkg/info/*\n\n# Apache environment, copied from\n# https://github.com/ijklim/laravel-benfords-law-app/blob/e9bf385dcaddb62ea466a7b245ab6e4ef708c313/docker/os/Dockerfile\nENV APACHE_DOCUMENT_ROOT=/var/www/html/public\nENV APACHE_RUN_USER www-data\nENV APACHE_RUN_GROUP www-data\nENV APACHE_PID_FILE /var/run/apache2.pid\nENV APACHE_RUN_DIR /var/run/apache2\nENV APACHE_LOCK_DIR /var/lock/apache2\nENV APACHE_LOG_DIR /var/log\nRUN ln -sf /dev/stdout /var/log/apache2-access.log\nRUN ln -sf /dev/stderr /var/log/apache2-error.log\nRUN mkdir -p $APACHE_RUN_DIR $APACHE_LOCK_DIR\n\nRUN a2enmod proxy\nRUN a2enmod proxy_http\nRUN a2enmod headers\n\nARG DATASETTE_REF\n\nRUN pip install \\\n    https://github.com/simonw/datasette/archive/${DATASETTE_REF}.zip \\\n    datasette-redirect-to-https datasette-debug-asgi\n\nADD 000-default.conf /etc/apache2/sites-enabled/000-default.conf\n\nWORKDIR /app\nRUN mkdir -p /app/html\nRUN echo '
    Demo is at /prefix/
    ' > /app/html/index.html\n\nADD https://latest.datasette.io/fixtures.db /app/fixtures.db\n\nEXPOSE 80\n\n# Dynamically build supervisord config since it includes $DATASETTE_REF:\nRUN echo \"[supervisord]\" >> /app/supervisord.conf\nRUN echo \"nodaemon=true\" >> /app/supervisord.conf\nRUN echo \"\" >> /app/supervisord.conf\nRUN echo \"[program:apache2]\" >> /app/supervisord.conf\nRUN echo \"command=apache2 -D FOREGROUND\" >> /app/supervisord.conf\nRUN echo \"stdout_logfile=/dev/stdout\" >> /app/supervisord.conf\nRUN echo \"stdout_logfile_maxbytes=0\" >> /app/supervisord.conf\nRUN echo \"\" >> /app/supervisord.conf\nRUN echo \"[program:datasette]\" >> /app/supervisord.conf\nRUN echo \"command=datasette /app/fixtures.db --setting base_url '/prefix/' --version-note '${DATASETTE_REF}' -h 0.0.0.0 -p 8001\" >> /app/supervisord.conf\nRUN echo \"stdout_logfile=/dev/stdout\" >> /app/supervisord.conf\nRUN echo \"stdout_logfile_maxbytes=0\" >> /app/supervisord.conf\n\nCMD [\"/usr/bin/supervisord\", \"-c\", \"/app/supervisord.conf\"]\n"
  },
  {
    "path": "demos/apache-proxy/README.md",
    "content": "# Datasette running behind an Apache proxy\n\nSee also [Running Datasette behind a proxy](https://docs.datasette.io/en/latest/deploying.html#running-datasette-behind-a-proxy)\n\nThis live demo is running at https://datasette-apache-proxy-demo.fly.dev/prefix/\n\nTo build locally, passing in a Datasette commit hash (or `main` for the main branch):\n\n    docker build -t datasette-apache-proxy-demo . \\\n      --build-arg DATASETTE_REF=c617e1769ea27e045b0f2907ef49a9a1244e577d\n\nThen run it like this:\n\n    docker run -p 5000:80 datasette-apache-proxy-demo\n\nAnd visit `http://localhost:5000/` or `http://localhost:5000/prefix/`\n\n## Deployment to Fly\n\nTo deploy to [Fly](https://fly.io/) first create an application there by running:\n\n    flyctl apps create --name datasette-apache-proxy-demo\n\nYou will need a different name, since I have already taken that one.\n\nThen run this command to deploy:\n\n    flyctl deploy --build-arg DATASETTE_REF=main\n\nThis uses `fly.toml` in this directory, which hard-codes the `datasette-apache-proxy-demo` name - so you would need to edit that file to match your application name before running this.\n\n## Deployment to Cloud Run\n\nDeployments to Cloud Run currently result in intermittent 503 errors and I'm not sure why, see [issue #1522](https://github.com/simonw/datasette/issues/1522).\n\nYou can deploy like this:\n\n    DATASETTE_REF=main ./deploy-to-cloud-run.sh\n"
  },
  {
    "path": "demos/apache-proxy/deploy-to-cloud-run.sh",
    "content": "#!/bin/bash\n# https://til.simonwillison.net/cloudrun/using-build-args-with-cloud-run\n\nif [[ -z \"$DATASETTE_REF\" ]]; then\n    echo \"Must provide DATASETTE_REF environment variable\" 1>&2\n    exit 1\nfi\n\nNAME=\"datasette-apache-proxy-demo\"\nPROJECT=$(gcloud config get-value project)\nIMAGE=\"gcr.io/$PROJECT/$NAME\"\n\n# Need YAML so we can set --build-arg\necho \"\nsteps:\n- name: 'gcr.io/cloud-builders/docker'\n  args: ['build', '-t', '$IMAGE', '.', '--build-arg', 'DATASETTE_REF=$DATASETTE_REF']\n- name: 'gcr.io/cloud-builders/docker'\n  args: ['push', '$IMAGE']\n\" > /tmp/cloudbuild.yml\n\ngcloud builds submit --config /tmp/cloudbuild.yml\n\nrm /tmp/cloudbuild.yml\n\ngcloud run deploy $NAME \\\n  --allow-unauthenticated \\\n  --platform=managed \\\n  --image $IMAGE \\\n  --port 80\n"
  },
  {
    "path": "demos/apache-proxy/fly.toml",
    "content": "app = \"datasette-apache-proxy-demo\"\n\nkill_signal = \"SIGINT\"\nkill_timeout = 5\nprocesses = []\n\n[env]\n\n[experimental]\n  allowed_public_ports = []\n  auto_rollback = true\n\n[[services]]\n  http_checks = []\n  internal_port = 80\n  processes = [\"app\"]\n  protocol = \"tcp\"\n  script_checks = []\n\n  [services.concurrency]\n    hard_limit = 25\n    soft_limit = 20\n    type = \"connections\"\n\n  [[services.ports]]\n    handlers = [\"http\"]\n    port = 80\n\n  [[services.ports]]\n    handlers = [\"tls\", \"http\"]\n    port = 443\n\n  [[services.tcp_checks]]\n    grace_period = \"1s\"\n    interval = \"15s\"\n    restart_limit = 0\n    timeout = \"2s\"\n"
  },
  {
    "path": "demos/plugins/example_js_manager_plugins.py",
    "content": "from datasette import hookimpl\n\n# Test command:\n# datasette fixtures.db \\ --plugins-dir=demos/plugins/\n#                       \\ --static static:demos/plugins/static\n\n# Create a set with view names that qualify for this JS, since plugins won't do anything on other pages\n# Same pattern as in Nteract data explorer\n# https://github.com/hydrosquall/datasette-nteract-data-explorer/blob/main/datasette_nteract_data_explorer/__init__.py#L77\nPERMITTED_VIEWS = {\"table\", \"query\", \"database\"}\n\n\n@hookimpl\ndef extra_js_urls(view_name):\n    print(view_name)\n    if view_name in PERMITTED_VIEWS:\n        return [\n            {\n                \"url\": \"/static/table-example-plugins.js\",\n            }\n        ]\n"
  },
  {
    "path": "demos/plugins/static/table-example-plugins.js",
    "content": "/**\n * Example usage of Datasette JS Manager API\n */\n\ndocument.addEventListener(\"datasette_init\", function (evt) {\n  const { detail: manager } = evt;\n  // === Demo plugins: remove before merge===\n  addPlugins(manager);\n});\n\n/**\n * Examples for to test datasette JS api\n */\nconst addPlugins = (manager) => {\n\n  manager.registerPlugin(\"column-name-plugin\", {\n    version: 0.1,\n    makeColumnActions: (columnMeta) => {\n      const { column } = columnMeta;\n\n      return [\n        {\n          label: \"Copy name to clipboard\",\n          onClick: (evt) => copyToClipboard(column),\n        },\n        {\n          label: \"Log column metadata to console\",\n          onClick: (evt) => console.log(column),\n        },\n      ];\n    },\n  });\n\n  manager.registerPlugin(\"panel-plugin-graphs\", {\n    version: 0.1,\n    makeAboveTablePanelConfigs: () => {\n      return [\n        {\n          id: 'first-panel',\n          label: \"First\",\n          render: node => {\n            const description = document.createElement('p');\n            description.innerText = 'Hello world';\n            node.appendChild(description);\n          }\n        },\n        {\n          id: 'second-panel',\n          label: \"Second\",\n          render: node => {\n            const iframe = document.createElement('iframe');\n            iframe.src = \"https://observablehq.com/embed/@d3/sortable-bar-chart?cell=viewof+order&cell=chart\";\n            iframe.width = 800;\n            iframe.height = 635;\n            iframe.frameborder = '0';\n            node.appendChild(iframe);\n          }\n        },\n      ];\n    },\n  });\n\n  manager.registerPlugin(\"panel-plugin-maps\", {\n    version: 0.1,\n    makeAboveTablePanelConfigs: () => {\n      return [\n        {\n          // ID only has to be unique within a plugin, manager namespaces for you\n          id: 'first-map-panel',\n          label: \"Map plugin\",\n          // datasette-vega, leafleft can provide a \"render\" function\n          render: node => node.innerHTML = \"Here sits a map\",\n        },\n        {\n          id: 'second-panel',\n          label: \"Image plugin\",\n          render: node => {\n            const img = document.createElement('img');\n            img.src = 'https://datasette.io/static/datasette-logo.svg'\n            node.appendChild(img);\n          },\n        }\n      ];\n    },\n  });\n\n  // Future: dispatch message to some other part of the page with CustomEvent API\n  // Could use to drive filter/sort query builder actions without  page refresh.\n}\n\n\n\nasync function copyToClipboard(str) {\n  try {\n    await navigator.clipboard.writeText(str);\n  } catch (err) {\n    /** Rejected - text failed to copy to the clipboard. Browsers didn't give permission */\n    console.error('Failed to copy: ', err);\n  }\n}\n"
  },
  {
    "path": "docs/.gitignore",
    "content": "_build\n"
  },
  {
    "path": "docs/Makefile",
    "content": "# Minimal makefile for Sphinx documentation\n#\n\n# You can set these variables from the command line.\nSPHINXOPTS    =\nSPHINXBUILD   = sphinx-build\nSPHINXPROJ    = Datasette\nSOURCEDIR     = .\nBUILDDIR      = _build\n\n# Put it first so that \"make\" without argument is like \"make help\".\nhelp:\n\t@$(SPHINXBUILD) -M help \"$(SOURCEDIR)\" \"$(BUILDDIR)\" $(SPHINXOPTS) $(O)\n\n.PHONY: help Makefile\n\n# Catch-all target: route all unknown targets to Sphinx using the new\n# \"make mode\" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).\n%: Makefile\n\t@$(SPHINXBUILD) -M $@ \"$(SOURCEDIR)\" \"$(BUILDDIR)\" $(SPHINXOPTS) $(O)\n\nlivehtml:\n\tsphinx-autobuild -b html \"$(SOURCEDIR)\" \"$(BUILDDIR)\" $(SPHINXOPTS) $(0)\n"
  },
  {
    "path": "docs/_static/css/custom.css",
    "content": "a.external {\n    overflow-wrap: anywhere;\n}\nbody[data-theme=\"dark\"] .sidebar-logo-container {\n    background-color: white;\n    padding: 5px;\n    opacity: 0.6;\n}\n"
  },
  {
    "path": "docs/_static/js/custom.js",
    "content": "jQuery(function ($) {\n  // Show banner linking to /stable/ if this is a /latest/ page\n  if (!/\\/latest\\//.test(location.pathname)) {\n    return;\n  }\n  var stableUrl = location.pathname.replace(\"/latest/\", \"/stable/\");\n  // Check it's not a 404\n  fetch(stableUrl, { method: \"HEAD\" }).then((response) => {\n    if (response.status == 200) {\n      var warning = $(\n        `
    \n           
    Note
    \n           
    \n             This documentation covers the development version of Datasette.
    \n             
    See this page for the current stable release.\n           
    \n        
    `\n      );\n      warning.find(\"a\").attr(\"href\", stableUrl);\n      $(\"article[role=main]\").prepend(warning);\n    }\n  });\n});\n"
  },
  {
    "path": "docs/_templates/base.html",
    "content": "{%- extends \"!base.html\" %}\n\n{% block site_meta %}\n{{ super() }}\n\n{% endblock %}\n\n{% block scripts %}\n{{ super() }}\n\n{% endblock %}\n"
  },
  {
    "path": "docs/_templates/sidebar/brand.html",
    "content": "
    \n  {% block brand_content %}\n  
    \n    \"Datasette\"\n  
    \n  {%- set nav_version = version %}\n  {% if READTHEDOCS and current_version %}\n    {%- set nav_version = current_version %}\n  {% endif %}\n  {% if nav_version %}\n    
    \n      {{ nav_version }}\n    
    \n  {% endif %}\n  {% endblock brand_content %}\n
    \n"
  },
  {
    "path": "docs/_templates/sidebar/navigation.html",
    "content": "
    \n  \n  {{ toctree(\n    collapse=True,\n    titles_only=False,\n    maxdepth=3,\n    includehidden=True,\n) }}\n
    "
  },
  {
    "path": "docs/authentication.rst",
    "content": ".. _authentication:\n\n================================\n Authentication and permissions\n================================\n\nDatasette doesn't require authentication by default. Any visitor to a Datasette instance can explore the full data and execute read-only SQL queries.\n\nDatasette can be configured to only allow authenticated users, or to control which databases, tables, and queries can be accessed by the public or by specific users. Datasette's plugin system can be used to add many different styles of authentication, such as user accounts, single sign-on or API keys.\n\n.. _authentication_actor:\n\nActors\n======\n\nThrough plugins, Datasette can support both authenticated users (with cookies) and authenticated API clients (via authentication tokens). The word \"actor\" is used to cover both of these cases.\n\nEvery request to Datasette has an associated actor value, available in the code as ``request.actor``. This can be ``None`` for unauthenticated requests, or a JSON compatible Python dictionary for authenticated users or API clients.\n\nThe actor dictionary can be any shape - the design of that data structure is left up to the plugins. Actors should always include a unique ``\"id\"`` string, as demonstrated by the \"root\" actor below.\n\nPlugins can use the :ref:`plugin_hook_actor_from_request` hook to implement custom logic for authenticating an actor based on the incoming HTTP request.\n\n.. _authentication_root:\n\nUsing the \"root\" actor\n----------------------\n\nDatasette currently leaves almost all forms of authentication to plugins - `datasette-auth-github `__ for example.\n\nThe one exception is the \"root\" account, which you can sign into while using Datasette on your local machine. The root user has **all permissions** - they can perform any action regardless of other permission rules.\n\nThe ``--root`` flag is designed for local development and testing. When you start Datasette with ``--root``, the root user automatically receives every permission, including:\n\n* All view permissions (``view-instance``, ``view-database``, ``view-table``, etc.)\n* All write permissions (``insert-row``, ``update-row``, ``delete-row``, ``create-table``, ``alter-table``, ``set-column-type``, ``drop-table``)\n* Debug permissions (``permissions-debug``, ``debug-menu``)\n* Any custom permissions defined by plugins\n\nIf you add explicit deny rules in ``datasette.yaml`` those can still block the\nroot actor from specific databases or tables.\n\nThe ``--root`` flag sets an internal ``root_enabled`` switch—without it, a signed-in user with ``{\"id\": \"root\"}`` is treated like any other actor.\n\nTo sign in as root, start Datasette using the ``--root`` command-line option, like this::\n\n    datasette --root\n\nDatasette will output a single-use-only login URL on startup::\n\n    http://127.0.0.1:8001/-/auth-token?token=786fc524e0199d70dc9a581d851f466244e114ca92f33aa3b42a139e9388daa7\n    INFO:     Started server process [25801]\n    INFO:     Waiting for application startup.\n    INFO:     Application startup complete.\n    INFO:     Uvicorn running on http://127.0.0.1:8001 (Press CTRL+C to quit)\n\nClick on that link and then visit ``http://127.0.0.1:8001/-/actor`` to confirm that you are authenticated as an actor that looks like this:\n\n.. code-block:: json\n\n    {\n        \"id\": \"root\"\n    }\n\n.. _authentication_permissions:\n\nPermissions\n===========\n\nDatasette's permissions system is built around SQL queries. Datasette and its plugins construct SQL queries to resolve the list of resources that an actor cas access.\n\nThe key question the permissions system answers is this:\n\n    Is this **actor** allowed to perform this **action**, optionally against this particular **resource**?\n\n**Actors** are :ref:`described above `.\n\nAn **action** is a string describing the action the actor would like to perform. A full list is :ref:`provided below ` - examples include ``view-table`` and ``execute-sql``.\n\nA **resource** is the item the actor wishes to interact with - for example a specific database or table. Some actions, such as ``permissions-debug``, are not associated with a particular resource.\n\nDatasette's built-in view actions (``view-database``, ``view-table`` etc) are allowed by Datasette's default configuration: unless you :ref:`configure additional permission rules ` unauthenticated users will be allowed to access content.\n\nOther actions, including those introduced by plugins, will default to *deny*.\n\n.. _authentication_default_deny:\n\nDenying all permissions by default\n----------------------------------\n\nBy default, Datasette allows unauthenticated access to view databases, tables, and execute SQL queries.\n\nYou may want to run Datasette in a mode where **all** access is denied by default, and you explicitly grant permissions only to authenticated users, either using the :ref:`--root mechanism ` or through :ref:`configuration file rules ` or plugins.\n\nUse the ``--default-deny`` command-line option to run Datasette in this mode::\n\n    datasette --default-deny data.db --root\n\nWith ``--default-deny`` enabled:\n\n* Anonymous users are denied access to view the instance, databases, tables, and queries\n* Authenticated users are also denied access unless they're explicitly granted permissions\n* The root user (when using ``--root``) still has access to everything\n* You can grant permissions using :ref:`configuration file rules ` or plugins\n\nFor example, to allow only a specific user to access your instance::\n\n    datasette --default-deny data.db --config datasette.yaml\n\nWhere ``datasette.yaml`` contains:\n\n.. code-block:: yaml\n\n    allow:\n      id: alice\n\nThis configuration will deny access to everyone except the user with ``id`` of ``alice``.\n\n.. _authentication_permissions_explained:\n\nHow permissions are resolved\n----------------------------\n\nDatasette performs permission checks using the internal :ref:`datasette_allowed`, method which accepts keyword arguments for ``action``, ``resource`` and an optional ``actor``. \n\n``resource`` should be an instance of the appropriate ``Resource`` subclass from :mod:`datasette.resources`—for example ``InstanceResource()``, ``DatabaseResource(database=\"...``)`` or ``TableResource(database=\"...\", table=\"...\")``. This defaults to ``InstanceResource()`` if not specified.\n\nWhen a check runs Datasette gathers allow/deny rules from multiple sources and\ncompiles them into a SQL query. The resulting query describes all of the\nresources an actor may access for that action, together with the reasons those\nresources were allowed or denied. The combined sources are:\n\n* ``allow`` blocks configured in :ref:`datasette.yaml `.\n* :ref:`Actor restrictions ` encoded into the actor dictionary or API token.\n* The \"root\" user shortcut when ``--root`` (or :attr:`Datasette.root_enabled `) is active, replying ``True`` to all permission chucks unless configuration rules deny them at a more specific level.\n* Any additional SQL provided by plugins implementing :ref:`plugin_hook_permission_resources_sql`.\n\nDatasette evaluates the SQL to determine if the requested ``resource`` is\nincluded. Explicit deny rules returned by configuration or plugins will block\naccess even if other rules allowed it.\n\n.. _authentication_permissions_allow:\n\nDefining permissions with \"allow\" blocks\n----------------------------------------\n\nOne way to define permissions in Datasette is to use an ``\"allow\"`` block :ref:`in the datasette.yaml file `. This is a JSON document describing which actors are allowed to perform an action against a specific resource.\n\nEach ``allow`` block is compiled into SQL and combined with any\n:ref:`plugin-provided rules ` to produce\nthe cascading allow/deny decisions that power :ref:`datasette_allowed`.\n\nThe most basic form of allow block is this (`allow demo `__, `deny demo `__):\n\n.. [[[cog\n    from metadata_doc import config_example\n    import textwrap\n    config_example(cog, textwrap.dedent(\n      \"\"\"\n        allow:\n          id: root\n        \"\"\").strip(),\n        \"YAML\", \"JSON\"\n      )\n.. ]]]\n\n.. tab:: YAML\n\n    .. code-block:: yaml\n\n        allow:\n          id: root\n\n.. tab:: JSON\n\n    .. code-block:: json\n\n        {\n          \"allow\": {\n            \"id\": \"root\"\n          }\n        }\n.. [[[end]]]\n\nThis will match any actors with an ``\"id\"`` property of ``\"root\"`` - for example, an actor that looks like this:\n\n.. code-block:: json\n\n    {\n        \"id\": \"root\",\n        \"name\": \"Root User\"\n    }\n\nAn allow block can specify \"deny all\" using ``false`` (`demo `__):\n\n.. [[[cog\n    from metadata_doc import config_example\n    import textwrap\n    config_example(cog, textwrap.dedent(\n      \"\"\"\n        allow: false\n        \"\"\").strip(),\n        \"YAML\", \"JSON\"\n      )\n.. ]]]\n\n.. tab:: YAML\n\n    .. code-block:: yaml\n\n        allow: false\n\n.. tab:: JSON\n\n    .. code-block:: json\n\n        {\n          \"allow\": false\n        }\n.. [[[end]]]\n\nAn ``\"allow\"`` of ``true`` allows all access (`demo `__):\n\n.. [[[cog\n    from metadata_doc import config_example\n    import textwrap\n    config_example(cog, textwrap.dedent(\n      \"\"\"\n        allow: true\n        \"\"\").strip(),\n        \"YAML\", \"JSON\"\n      )\n.. ]]]\n\n.. tab:: YAML\n\n    .. code-block:: yaml\n\n        allow: true\n\n.. tab:: JSON\n\n    .. code-block:: json\n\n        {\n          \"allow\": true\n        }\n.. [[[end]]]\n\nAllow keys can provide a list of values. These will match any actor that has any of those values (`allow demo `__, `deny demo `__):\n\n.. [[[cog\n    from metadata_doc import config_example\n    import textwrap\n    config_example(cog, textwrap.dedent(\n      \"\"\"\n        allow:\n          id:\n          - simon\n          - cleopaws\n        \"\"\").strip(),\n        \"YAML\", \"JSON\"\n      )\n.. ]]]\n\n.. tab:: YAML\n\n    .. code-block:: yaml\n\n        allow:\n          id:\n          - simon\n          - cleopaws\n\n.. tab:: JSON\n\n    .. code-block:: json\n\n        {\n          \"allow\": {\n            \"id\": [\n              \"simon\",\n              \"cleopaws\"\n            ]\n          }\n        }\n.. [[[end]]]\n\nThis will match any actor with an ``\"id\"`` of either ``\"simon\"`` or ``\"cleopaws\"``.\n\nActors can have properties that feature a list of values. These will be matched against the list of values in an allow block. Consider the following actor:\n\n.. code-block:: json\n\n      {\n          \"id\": \"simon\",\n          \"roles\": [\"staff\", \"developer\"]\n      }\n\nThis allow block will provide access to any actor that has ``\"developer\"`` as one of their roles (`allow demo `__, `deny demo `__):\n\n.. [[[cog\n    from metadata_doc import config_example\n    import textwrap\n    config_example(cog, textwrap.dedent(\n      \"\"\"\n        allow:\n          roles:\n          - developer\n        \"\"\").strip(),\n        \"YAML\", \"JSON\"\n      )\n.. ]]]\n\n.. tab:: YAML\n\n    .. code-block:: yaml\n\n        allow:\n          roles:\n          - developer\n\n.. tab:: JSON\n\n    .. code-block:: json\n\n        {\n          \"allow\": {\n            \"roles\": [\n              \"developer\"\n            ]\n          }\n        }\n.. [[[end]]]\n\nNote that \"roles\" is not a concept that is baked into Datasette - it's a convention that plugins can choose to implement and act on.\n\nIf you want to provide access to any actor with a value for a specific key, use ``\"*\"``. For example, to match any logged-in user specify the following (`allow demo `__, `deny demo `__):\n\n.. [[[cog\n    from metadata_doc import config_example\n    import textwrap\n    config_example(cog, textwrap.dedent(\n      \"\"\"\n        allow:\n          id: \"*\"\n        \"\"\").strip(),\n        \"YAML\", \"JSON\"\n      )\n.. ]]]\n\n.. tab:: YAML\n\n    .. code-block:: yaml\n\n        allow:\n          id: \"*\"\n\n.. tab:: JSON\n\n    .. code-block:: json\n\n        {\n          \"allow\": {\n            \"id\": \"*\"\n          }\n        }\n.. [[[end]]]\n\nYou can specify that only unauthenticated actors (from anonymous HTTP requests) should be allowed access using the special ``\"unauthenticated\": true`` key in an allow block (`allow demo `__, `deny demo `__):\n\n.. [[[cog\n    from metadata_doc import config_example\n    import textwrap\n    config_example(cog, textwrap.dedent(\n      \"\"\"\n        allow:\n          unauthenticated: true\n        \"\"\").strip(),\n        \"YAML\", \"JSON\"\n      )\n.. ]]]\n\n.. tab:: YAML\n\n    .. code-block:: yaml\n\n        allow:\n          unauthenticated: true\n\n.. tab:: JSON\n\n    .. code-block:: json\n\n        {\n          \"allow\": {\n            \"unauthenticated\": true\n          }\n        }\n.. [[[end]]]\n\nAllow keys act as an \"or\" mechanism. An actor will be able to execute the query if any of their JSON properties match any of the values in the corresponding lists in the ``allow`` block. The following block will allow users with either a ``role`` of ``\"ops\"`` OR users who have an ``id`` of ``\"simon\"`` or ``\"cleopaws\"``:\n\n.. [[[cog\n    from metadata_doc import config_example\n    import textwrap\n    config_example(cog, textwrap.dedent(\n      \"\"\"\n        allow:\n          id:\n          - simon\n          - cleopaws\n          role: ops\n        \"\"\").strip(),\n        \"YAML\", \"JSON\"\n      )\n.. ]]]\n\n.. tab:: YAML\n\n    .. code-block:: yaml\n\n        allow:\n          id:\n          - simon\n          - cleopaws\n          role: ops\n\n.. tab:: JSON\n\n    .. code-block:: json\n\n        {\n          \"allow\": {\n            \"id\": [\n              \"simon\",\n              \"cleopaws\"\n            ],\n            \"role\": \"ops\"\n          }\n        }\n.. [[[end]]]\n\n`Demo for cleopaws `__, `demo for ops role `__, `demo for an actor matching neither rule `__.\n\n.. _AllowDebugView:\n\nThe /-/allow-debug tool\n-----------------------\n\nThe ``/-/allow-debug`` tool lets you try out different  ``\"action\"`` blocks against different ``\"actor\"`` JSON objects. You can try that out here: https://latest.datasette.io/-/allow-debug\n\n.. _authentication_permissions_config:\n\nAccess permissions in ``datasette.yaml``\n========================================\n\nThere are two ways to configure permissions using ``datasette.yaml`` (or ``datasette.json``).\n\nFor simple visibility permissions you can use ``\"allow\"`` blocks in the root, database, table and query sections.\n\nFor other permissions you can use a ``\"permissions\"`` block, described :ref:`in the next section `.\n\nYou can limit who is allowed to view different parts of your Datasette instance using ``\"allow\"`` keys in your :ref:`configuration`.\n\nYou can control the following:\n\n* Access to the entire Datasette instance\n* Access to specific databases\n* Access to specific tables and views\n* Access to specific :ref:`canned_queries`\n\nIf a user has permission to view a table they will be able to view that table, independent of if they have permission to view the database or instance that the table exists within.\n\n.. _authentication_permissions_instance:\n\nAccess to an instance\n---------------------\n\nHere's how to restrict access to your entire Datasette instance to just the ``\"id\": \"root\"`` user:\n\n.. [[[cog\n    from metadata_doc import config_example\n    config_example(cog, \"\"\"\n        title: My private Datasette instance\n        allow:\n          id: root\n      \"\"\")\n.. ]]]\n\n.. tab:: datasette.yaml\n\n    .. code-block:: yaml\n\n\n            title: My private Datasette instance\n            allow:\n              id: root\n  \n\n.. tab:: datasette.json\n\n    .. code-block:: json\n\n        {\n          \"title\": \"My private Datasette instance\",\n          \"allow\": {\n            \"id\": \"root\"\n          }\n        }\n.. [[[end]]]\n\nTo deny access to all users, you can use ``\"allow\": false``:\n\n.. [[[cog\n    config_example(cog, \"\"\"\n        title: My entirely inaccessible instance\n        allow: false\n    \"\"\")\n.. ]]]\n\n.. tab:: datasette.yaml\n\n    .. code-block:: yaml\n\n\n            title: My entirely inaccessible instance\n            allow: false\n\n\n.. tab:: datasette.json\n\n    .. code-block:: json\n\n        {\n          \"title\": \"My entirely inaccessible instance\",\n          \"allow\": false\n        }\n.. [[[end]]]\n\nOne reason to do this is if you are using a Datasette plugin - such as `datasette-permissions-sql `__ - to control permissions instead.\n\n.. _authentication_permissions_database:\n\nAccess to specific databases\n----------------------------\n\nTo limit access to a specific ``private.db`` database to just authenticated users, use the ``\"allow\"`` block like this:\n\n.. [[[cog\n    config_example(cog, \"\"\"\n        databases:\n          private:\n            allow:\n              id: \"*\"\n    \"\"\")\n.. ]]]\n\n.. tab:: datasette.yaml\n\n    .. code-block:: yaml\n\n\n            databases:\n              private:\n                allow:\n                  id: \"*\"\n\n\n.. tab:: datasette.json\n\n    .. code-block:: json\n\n        {\n          \"databases\": {\n            \"private\": {\n              \"allow\": {\n                \"id\": \"*\"\n              }\n            }\n          }\n        }\n.. [[[end]]]\n\n.. _authentication_permissions_table:\n\nAccess to specific tables and views\n-----------------------------------\n\nTo limit access to the ``users`` table in your ``bakery.db`` database:\n\n.. [[[cog\n    config_example(cog, \"\"\"\n        databases:\n          bakery:\n            tables:\n              users:\n                allow:\n                  id: '*'\n    \"\"\")\n.. ]]]\n\n.. tab:: datasette.yaml\n\n    .. code-block:: yaml\n\n\n            databases:\n              bakery:\n                tables:\n                  users:\n                    allow:\n                      id: '*'\n\n\n.. tab:: datasette.json\n\n    .. code-block:: json\n\n        {\n          \"databases\": {\n            \"bakery\": {\n              \"tables\": {\n                \"users\": {\n                  \"allow\": {\n                    \"id\": \"*\"\n                  }\n                }\n              }\n            }\n          }\n        }\n.. [[[end]]]\n\nThis works for SQL views as well - you can list their names in the ``\"tables\"`` block above in the same way as regular tables.\n\n.. warning::\n    Restricting access to tables and views in this way will NOT prevent users from querying them using arbitrary SQL queries, `like this `__ for example.\n\n    If you are restricting access to specific tables you should also use the ``\"allow_sql\"`` block to prevent users from bypassing the limit with their own SQL queries - see :ref:`authentication_permissions_execute_sql`.\n\n.. _authentication_permissions_query:\n\nAccess to specific canned queries\n---------------------------------\n\n:ref:`canned_queries` allow you to configure named SQL queries in your ``datasette.yaml`` that can be executed by users. These queries can be set up to both read and write to the database, so controlling who can execute them can be important.\n\nTo limit access to the ``add_name`` canned query in your ``dogs.db`` database to just the :ref:`root user`:\n\n.. [[[cog\n    config_example(cog, \"\"\"\n        databases:\n          dogs:\n            queries:\n              add_name:\n                sql: INSERT INTO names (name) VALUES (:name)\n                write: true\n                allow:\n                  id:\n                  - root\n    \"\"\")\n.. ]]]\n\n.. tab:: datasette.yaml\n\n    .. code-block:: yaml\n\n\n            databases:\n              dogs:\n                queries:\n                  add_name:\n                    sql: INSERT INTO names (name) VALUES (:name)\n                    write: true\n                    allow:\n                      id:\n                      - root\n\n\n.. tab:: datasette.json\n\n    .. code-block:: json\n\n        {\n          \"databases\": {\n            \"dogs\": {\n              \"queries\": {\n                \"add_name\": {\n                  \"sql\": \"INSERT INTO names (name) VALUES (:name)\",\n                  \"write\": true,\n                  \"allow\": {\n                    \"id\": [\n                      \"root\"\n                    ]\n                  }\n                }\n              }\n            }\n          }\n        }\n.. [[[end]]]\n\n.. _authentication_permissions_execute_sql:\n\nControlling the ability to execute arbitrary SQL\n------------------------------------------------\n\nDatasette defaults to allowing any site visitor to execute their own custom SQL queries, for example using the form on `the database page `__ or by appending a ``?_where=`` parameter to the table page `like this `__.\n\nAccess to this ability is controlled by the :ref:`actions_execute_sql` permission.\n\nThe easiest way to disable arbitrary SQL queries is using the :ref:`default_allow_sql setting ` when you first start Datasette running.\n\nYou can alternatively use an ``\"allow_sql\"`` block to control who is allowed to execute arbitrary SQL queries.\n\nTo prevent any user from executing arbitrary SQL queries, use this:\n\n.. [[[cog\n    config_example(cog, \"\"\"\n        allow_sql: false\n    \"\"\")\n.. ]]]\n\n.. tab:: datasette.yaml\n\n    .. code-block:: yaml\n\n\n            allow_sql: false\n\n\n.. tab:: datasette.json\n\n    .. code-block:: json\n\n        {\n          \"allow_sql\": false\n        }\n.. [[[end]]]\n\nTo enable just the :ref:`root user` to execute SQL for all databases in your instance, use the following:\n\n.. [[[cog\n    config_example(cog, \"\"\"\n        allow_sql:\n          id: root\n    \"\"\")\n.. ]]]\n\n.. tab:: datasette.yaml\n\n    .. code-block:: yaml\n\n\n            allow_sql:\n              id: root\n\n\n.. tab:: datasette.json\n\n    .. code-block:: json\n\n        {\n          \"allow_sql\": {\n            \"id\": \"root\"\n          }\n        }\n.. [[[end]]]\n\nTo limit this ability for just one specific database, use this:\n\n.. [[[cog\n    config_example(cog, \"\"\"\n        databases:\n          mydatabase:\n            allow_sql:\n              id: root\n    \"\"\")\n.. ]]]\n\n.. tab:: datasette.yaml\n\n    .. code-block:: yaml\n\n\n            databases:\n              mydatabase:\n                allow_sql:\n                  id: root\n\n\n.. tab:: datasette.json\n\n    .. code-block:: json\n\n        {\n          \"databases\": {\n            \"mydatabase\": {\n              \"allow_sql\": {\n                \"id\": \"root\"\n              }\n            }\n          }\n        }\n.. [[[end]]]\n\n.. _authentication_permissions_other:\n\nOther permissions in ``datasette.yaml``\n=======================================\n\nFor all other permissions, you can use one or more ``\"permissions\"`` blocks in your ``datasette.yaml`` configuration file.\n\nTo grant access to the :ref:`permissions debug tool ` to all signed in users, you can grant ``permissions-debug`` to any actor with an ``id`` matching the wildcard ``*`` by adding this a the root of your configuration:\n\n.. [[[cog\n    config_example(cog, \"\"\"\n        permissions:\n          debug-menu:\n            id: '*'\n    \"\"\")\n.. ]]]\n\n.. tab:: datasette.yaml\n\n    .. code-block:: yaml\n\n\n            permissions:\n              debug-menu:\n                id: '*'\n\n\n.. tab:: datasette.json\n\n    .. code-block:: json\n\n        {\n          \"permissions\": {\n            \"debug-menu\": {\n              \"id\": \"*\"\n            }\n          }\n        }\n.. [[[end]]]\n\nTo grant ``create-table`` to the user with ``id`` of ``editor`` for the ``docs`` database:\n\n.. [[[cog\n    config_example(cog, \"\"\"\n        databases:\n          docs:\n            permissions:\n              create-table:\n                id: editor\n    \"\"\")\n.. ]]]\n\n.. tab:: datasette.yaml\n\n    .. code-block:: yaml\n\n\n            databases:\n              docs:\n                permissions:\n                  create-table:\n                    id: editor\n\n\n.. tab:: datasette.json\n\n    .. code-block:: json\n\n        {\n          \"databases\": {\n            \"docs\": {\n              \"permissions\": {\n                \"create-table\": {\n                  \"id\": \"editor\"\n                }\n              }\n            }\n          }\n        }\n.. [[[end]]]\n\nOther table-scoped write permissions, including ``set-column-type``, can be configured in the same place.\n\nAnd for ``insert-row`` against the ``reports`` table in that ``docs`` database:\n\n.. [[[cog\n    config_example(cog, \"\"\"\n        databases:\n          docs:\n            tables:\n              reports:\n                permissions:\n                  insert-row:\n                    id: editor\n    \"\"\")\n.. ]]]\n\n.. tab:: datasette.yaml\n\n    .. code-block:: yaml\n\n\n            databases:\n              docs:\n                tables:\n                  reports:\n                    permissions:\n                      insert-row:\n                        id: editor\n\n\n.. tab:: datasette.json\n\n    .. code-block:: json\n\n        {\n          \"databases\": {\n            \"docs\": {\n              \"tables\": {\n                \"reports\": {\n                  \"permissions\": {\n                    \"insert-row\": {\n                      \"id\": \"editor\"\n                    }\n                  }\n                }\n              }\n            }\n          }\n        }\n.. [[[end]]]\n\nThe :ref:`permissions debug tool ` can be useful for helping test permissions that you have configured in this way.\n\n.. _CreateTokenView:\n\nAPI Tokens\n==========\n\nDatasette includes a default mechanism for generating API tokens that can be used to authenticate requests.\n\nAuthenticated users can create new API tokens using a form on the ``/-/create-token`` page.\n\nTokens created in this way can be further restricted to only allow access to specific actions, or to limit those actions to specific databases, tables or queries.\n\nCreated tokens can then be passed in the ``Authorization: Bearer $token`` header of HTTP requests to Datasette.\n\nA token created by a user will include that user's ``\"id\"`` in the token payload, so any permissions granted to that user based on their ID can be made available to the token as well.\n\nWhen one of these a token accompanies a request, the actor for that request will have the following shape:\n\n.. code-block:: json\n\n    {\n        \"id\": \"user_id\",\n        \"token\": \"dstok\",\n        \"token_expires\": 1667717426\n    }\n\nThe ``\"id\"`` field duplicates the ID of the actor who first created the token.\n\nThe ``\"token\"`` field identifies that this actor was authenticated using a Datasette signed token (``dstok``).\n\nThe ``\"token_expires\"`` field, if present, indicates that the token will expire after that integer timestamp.\n\nThe ``/-/create-token`` page cannot be accessed by actors that are authenticated with a ``\"token\": \"some-value\"`` property. This is to prevent API tokens from being used to create more tokens.\n\nDatasette plugins that implement their own form of API token authentication should follow this convention.\n\nYou can disable the signed token feature entirely using the :ref:`allow_signed_tokens ` setting.\n\n.. _authentication_cli_create_token:\n\ndatasette create-token\n----------------------\n\nYou can also create tokens on the command line using the ``datasette create-token`` command.\n\nThis command takes one required argument - the ID of the actor to be associated with the created token.\n\nYou can specify a ``-e/--expires-after`` option in seconds. If omitted, the token will never expire.\n\nThe command will sign the token using the ``DATASETTE_SECRET`` environment variable, if available. You can also pass the secret using the ``--secret`` option.\n\nThis means you can run the command locally to create tokens for use with a deployed Datasette instance, provided you know that instance's secret.\n\nTo create a token for the ``root`` actor that will expire in one hour::\n\n    datasette create-token root --expires-after 3600\n\nTo create a token that never expires using a specific secret::\n\n    datasette create-token root --secret my-secret-goes-here\n\n.. _authentication_cli_create_token_restrict:\n\nRestricting the actions that a token can perform\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nTokens created using ``datasette create-token ACTOR_ID`` will inherit all of the permissions of the actor that they are associated with.\n\nYou can pass additional options to create tokens that are restricted to a subset of that actor's permissions.\n\nTo restrict the token to just specific permissions against all available databases, use the ``--all`` option::\n\n    datasette create-token root --all insert-row --all update-row\n\nThis option can be passed as many times as you like. In the above example the token will only be allowed to insert and update rows.\n\nYou can also restrict permissions such that they can only be used within specific databases::\n\n    datasette create-token root --database mydatabase insert-row\n\nThe resulting token will only be able to insert rows, and only to tables in the ``mydatabase`` database.\n\nFinally, you can restrict permissions to individual resources - tables, SQL views and :ref:`named queries ` - within a specific database::\n\n    datasette create-token root --resource mydatabase mytable insert-row\n\nThese options have short versions: ``-a`` for ``--all``, ``-d`` for ``--database`` and ``-r`` for ``--resource``.\n\nYou can add ``--debug`` to see a JSON representation of the token that has been created. Here's a full example::\n\n    datasette create-token root \\\n        --secret mysecret \\\n        --all view-instance \\\n        --all view-table \\\n        --database docs view-query \\\n        --resource docs documents insert-row \\\n        --resource docs documents update-row \\\n        --debug\n\nThis example outputs the following::\n\n    dstok_.eJxFizEKgDAMRe_y5w4qYrFXERGxDkVsMI0uxbubdjFL8l_ez1jhwEQCA6Fjjxp90qtkuHawzdjYrh8MFobLxZ_wBH0_gtnAF-hpS5VfmF8D_lnd97lHqUJgLd6sls4H1qwlhA.nH_7RecYHj5qSzvjhMU95iy0Xlc\n\n    Decoded:\n\n    {\n      \"a\": \"root\",\n      \"token\": \"dstok\",\n      \"t\": 1670907246,\n      \"_r\": {\n        \"a\": [\n          \"vi\",\n          \"vt\"\n        ],\n        \"d\": {\n          \"docs\": [\n            \"vq\"\n          ]\n        },\n        \"r\": {\n          \"docs\": {\n            \"documents\": [\n              \"ir\",\n              \"ur\"\n            ]\n          }\n        }\n      }\n    }\n\nRestrictions act as an allowlist layered on top of the actor's existing\npermissions. They can only remove access the actor would otherwise have—they\ncannot grant new access. If the underlying actor is denied by ``allow`` rules in\n``datasette.yaml`` or by a plugin, a token that lists that resource in its\n``\"_r\"`` section will still be denied.\n\nTo create tokens with restrictions in Python code, use the :ref:`TokenRestrictions ` builder and pass it to :ref:`datasette.create_token() `.\n\n.. _permissions_plugins:\n\nChecking permissions in plugins\n===============================\n\nDatasette plugins can check if an actor has permission to perform an action using :ref:`datasette_allowed`—for example::\n\n    from datasette.resources import TableResource\n\n    can_edit = await datasette.allowed(\n        action=\"update-row\",\n        resource=TableResource(database=\"fixtures\", table=\"facetable\"),\n        actor=request.actor,\n    )\n\nUse :ref:`datasette_ensure_permission` when you need to enforce a permission and\nraise a ``Forbidden`` error automatically.\n\nPlugins that define new operations should return :class:`~datasette.permissions.Action`\nobjects from :ref:`plugin_register_actions` and can supply additional allow/deny\nrules by returning :class:`~datasette.permissions.PermissionSQL` objects from the\n:ref:`plugin_hook_permission_resources_sql` hook. Those rules are merged with\nconfiguration ``allow`` blocks and actor restrictions to determine the final\nresult for each check.\n\n.. _authentication_actor_matches_allow:\n\nactor_matches_allow()\n=====================\n\nPlugins that wish to implement this same ``\"allow\"`` block permissions scheme can take advantage of the ``datasette.utils.actor_matches_allow(actor, allow)`` function:\n\n.. code-block:: python\n\n    from datasette.utils import actor_matches_allow\n\n    actor_matches_allow({\"id\": \"root\"}, {\"id\": \"*\"})\n    # returns True\n\nThe currently authenticated actor is made available to plugins as ``request.actor``.\n\n.. _PermissionsDebugView:\n\nPermissions debug tools\n=======================\n\nThe debug tool at ``/-/permissions`` is available to any actor with the ``permissions-debug`` permission. By default this is just the :ref:`authenticated root user ` but you can open it up to all users by starting Datasette like this::\n\n    datasette -s permissions.permissions-debug true data.db\n\nThe page shows the permission checks that have been carried out by the Datasette instance.\n\nIt also provides an interface for running hypothetical permission checks against a hypothetical actor. This is a useful way of confirming that your configured permissions work in the way you expect.\n\nThis is designed to help administrators and plugin authors understand exactly how permission checks are being carried out, in order to effectively configure Datasette's permission system.\n\n.. _AllowedResourcesView:\n\nAllowed resources view\n----------------------\n\nThe ``/-/allowed`` endpoint displays resources that the current actor can access for a specified ``action``.\n\nThis endpoint provides an interactive HTML form interface. Add ``.json`` to the URL path (e.g. ``/-/allowed.json``) to get the raw JSON response instead.\n\nPass ``?action=view-table`` (or another action) to select the action. Optional ``parent=`` and ``child=`` query parameters can narrow the results to a specific database/table pair.\n\nThis endpoint is publicly accessible to help users understand their own permissions. The potentially sensitive ``reason`` field is only shown to users with the ``permissions-debug`` permission - it shows the plugins and explanatory reasons that were responsible for each decision.\n\n.. _PermissionRulesView:\n\nPermission rules view\n---------------------\n\nThe ``/-/rules`` endpoint displays all permission rules (both allow and deny) for each candidate resource for the requested action.\n\nThis endpoint provides an interactive HTML form interface. Add ``.json`` to the URL path (e.g. ``/-/rules.json?action=view-table``) to get the raw JSON response instead.\n\nPass ``?action=`` as a query parameter to specify which action to check.\n\nThis endpoint requires the ``permissions-debug`` permission.\n\n.. _PermissionCheckView:\n\nPermission check view\n---------------------\n\nThe ``/-/check`` endpoint evaluates a single action/resource pair and returns information indicating whether the access was allowed along with diagnostic information.\n\nThis endpoint provides an interactive HTML form interface. Add ``.json`` to the URL path (e.g. ``/-/check.json?action=view-instance``) to get the raw JSON response instead.\n\nPass ``?action=`` to specify the action to check, and optional ``?parent=`` and ``?child=`` parameters to specify the resource.\n\n.. _authentication_ds_actor:\n\nThe ds_actor cookie\n===================\n\nDatasette includes a default authentication plugin which looks for a signed ``ds_actor`` cookie containing a JSON actor dictionary. This is how the :ref:`root actor ` mechanism works.\n\nAuthentication plugins can set signed ``ds_actor`` cookies themselves like so:\n\n.. code-block:: python\n\n    response = Response.redirect(\"/\")\n    datasette.set_actor_cookie(response, {\"id\": \"cleopaws\"})\n\nThe shape of data encoded in the cookie is as follows:\n\n.. code-block:: json\n\n    {\n      \"a\": {\n        \"id\": \"cleopaws\"\n      }\n    }\n\nTo implement logout in a plugin, use the ``delete_actor_cookie()`` method:\n\n.. code-block:: python\n\n    response = Response.redirect(\"/\")\n    datasette.delete_actor_cookie(response)\n\n.. _authentication_ds_actor_expiry:\n\nIncluding an expiry time\n------------------------\n\n``ds_actor`` cookies can optionally include a signed expiry timestamp, after which the cookies will no longer be valid. Authentication plugins may chose to use this mechanism to limit the lifetime of the cookie. For example, if a plugin implements single-sign-on against another source it may decide to set short-lived cookies so that if the user is removed from the SSO system their existing Datasette cookies will stop working shortly afterwards.\n\nTo include an expiry pass ``expire_after=`` to ``datasette.set_actor_cookie()`` with a number of seconds. For example, to expire in 24 hours:\n\n.. code-block:: python\n\n    response = Response.redirect(\"/\")\n    datasette.set_actor_cookie(\n        response, {\"id\": \"cleopaws\"}, expire_after=60 * 60 * 24\n    )\n\nThe resulting cookie will encode data that looks something like this:\n\n.. code-block:: json\n\n    {\n      \"a\": {\n        \"id\": \"cleopaws\"\n      },\n      \"e\": \"1jjSji\"\n    }\n\n.. _LogoutView:\n\nThe /-/logout page\n------------------\n\nThe page at ``/-/logout`` provides the ability to log out of a ``ds_actor`` cookie authentication session.\n\n.. _actions:\n\nBuilt-in actions\n================\n\nThis section lists all of the permission checks that are carried out by Datasette core, along with the ``resource`` if it was passed.\n\n.. _actions_view_instance:\n\nview-instance\n-------------\n\nTop level permission - Actor is allowed to view any pages within this instance, starting at https://latest.datasette.io/\n\n.. _actions_view_database:\n\nview-database\n-------------\n\nActor is allowed to view a database page, e.g. https://latest.datasette.io/fixtures\n\n``resource`` - ``datasette.permissions.DatabaseResource(database)``\n    ``database`` is the name of the database (string)\n\n.. _actions_view_database_download:\n\nview-database-download\n----------------------\n\nActor is allowed to download a database, e.g. https://latest.datasette.io/fixtures.db\n\n``resource`` - ``datasette.resources.DatabaseResource(database)``\n    ``database`` is the name of the database (string)\n\n.. _actions_view_table:\n\nview-table\n----------\n\nActor is allowed to view a table (or view) page, e.g. https://latest.datasette.io/fixtures/complex_foreign_keys\n\n``resource`` - ``datasette.resources.TableResource(database, table)``\n    ``database`` is the name of the database (string)\n\n    ``table`` is the name of the table (string)\n\n.. _actions_view_query:\n\nview-query\n----------\n\nActor is allowed to view (and execute) a :ref:`canned query ` page, e.g. https://latest.datasette.io/fixtures/pragma_cache_size - this includes executing :ref:`canned_queries_writable`.\n\n``resource`` - ``datasette.resources.QueryResource(database, query)``\n    ``database`` is the name of the database (string)\n    \n    ``query`` is the name of the canned query (string)\n\n.. _actions_insert_row:\n\ninsert-row\n----------\n\nActor is allowed to insert rows into a table.\n\n``resource`` - ``datasette.resources.TableResource(database, table)``\n    ``database`` is the name of the database (string)\n\n    ``table`` is the name of the table (string)\n\n.. _actions_delete_row:\n\ndelete-row\n----------\n\nActor is allowed to delete rows from a table.\n\n``resource`` - ``datasette.resources.TableResource(database, table)``\n    ``database`` is the name of the database (string)\n\n    ``table`` is the name of the table (string)\n\n.. _actions_update_row:\n\nupdate-row\n----------\n\nActor is allowed to update rows in a table.\n\n``resource`` - ``datasette.resources.TableResource(database, table)``\n    ``database`` is the name of the database (string)\n\n    ``table`` is the name of the table (string)\n\n.. _actions_create_table:\n\ncreate-table\n------------\n\nActor is allowed to create a database table.\n\n``resource`` - ``datasette.resources.DatabaseResource(database)``\n    ``database`` is the name of the database (string)\n\n.. _actions_alter_table:\n\nalter-table\n-----------\n\nActor is allowed to alter a database table.\n\n``resource`` - ``datasette.resources.TableResource(database, table)``\n    ``database`` is the name of the database (string)\n\n    ``table`` is the name of the table (string)\n\n.. _actions_set_column_type:\n\nset-column-type\n---------------\n\nActor is allowed to set assigned :ref:`column types ` for columns in a table.\n\n``resource`` - ``datasette.resources.TableResource(database, table)``\n    ``database`` is the name of the database (string)\n\n    ``table`` is the name of the table (string)\n\n.. _actions_drop_table:\n\ndrop-table\n----------\n\nActor is allowed to drop a database table.\n\n``resource`` - ``datasette.resources.TableResource(database, table)``\n    ``database`` is the name of the database (string)\n\n    ``table`` is the name of the table (string)\n\n.. _actions_execute_sql:\n\nexecute-sql\n-----------\n\nActor is allowed to run arbitrary SQL queries against a specific database, e.g. https://latest.datasette.io/fixtures/-/query?sql=select+100\n\n``resource`` - ``datasette.resources.DatabaseResource(database)``\n    ``database`` is the name of the database (string)\n\nSee also :ref:`the default_allow_sql setting `.\n\n.. _actions_permissions_debug:\n\npermissions-debug\n-----------------\n\nActor is allowed to view the ``/-/permissions`` debug tools.\n\n.. _actions_debug_menu:\n\ndebug-menu\n----------\n\nControls if the various debug pages are displayed in the navigation menu.\n"
  },
  {
    "path": "docs/auto-build.sh",
    "content": "sphinx-autobuild . _build/html\n"
  },
  {
    "path": "docs/binary_data.rst",
    "content": ".. _binary:\n\n=============\n Binary data\n=============\n\nSQLite tables can contain binary data in ``BLOB`` columns.\n\nDatasette includes special handling for these binary values. The Datasette interface detects binary values and provides a link to download their content, for example on https://latest.datasette.io/fixtures/binary_data\n\n.. image:: https://raw.githubusercontent.com/simonw/datasette-screenshots/0.62/binary-data.png\n   :width: 311px\n   :alt: Screenshot showing download links next to binary data in the table view\n\nBinary data is represented in ``.json`` exports using Base64 encoding.\n\nhttps://latest.datasette.io/fixtures/binary_data.json?_shape=array\n\n.. code-block:: json\n\n    [\n        {\n            \"rowid\": 1,\n            \"data\": {\n                \"$base64\": true,\n                \"encoded\": \"FRwCx60F/g==\"\n            }\n        },\n        {\n            \"rowid\": 2,\n            \"data\": {\n                \"$base64\": true,\n                \"encoded\": \"FRwDx60F/g==\"\n            }\n        },\n        {\n            \"rowid\": 3,\n            \"data\": null\n        }\n    ]\n\n.. _binary_linking:\n\nLinking to binary downloads\n---------------------------\n\nThe ``.blob`` output format is used to return binary data. It requires a ``_blob_column=`` query string argument specifying which BLOB column should be downloaded, for example:\n\nhttps://latest.datasette.io/fixtures/binary_data/1.blob?_blob_column=data\n\nThis output format can also be used to return binary data from an arbitrary SQL query. Since such queries do not specify an exact row, an additional ``?_blob_hash=`` parameter can be used to specify the SHA-256 hash of the value that is being linked to.\n\nConsider the query ``select data from binary_data`` - `demonstrated here `__.\n\nThat page links to the binary value downloads. Those links look like this:\n\nhttps://latest.datasette.io/fixtures.blob?sql=select+data+from+binary_data&_blob_column=data&_blob_hash=f3088978da8f9aea479ffc7f631370b968d2e855eeb172bea7f6c7a04262bb6d\n\nThese ``.blob`` links are also returned in the ``.csv`` exports Datasette provides for binary tables and queries, since the CSV format does not have a mechanism for representing binary data.\n\nBinary plugins\n--------------\n\nSeveral Datasette plugins are available that change the way Datasette treats binary data.\n\n- `datasette-render-binary `__ modifies Datasette's default interface to show an automatic guess at what type of binary data is being stored, along with a visual representation of the binary value that displays ASCII strings directly in the interface.\n- `datasette-render-images `__ detects common image formats and renders them as images directly in the Datasette interface.\n- `datasette-media `__ allows Datasette interfaces to be configured to serve binary files from configured SQL queries, and includes the ability to resize images directly before serving them.\n"
  },
  {
    "path": "docs/changelog.rst",
    "content": ".. _changelog:\n\n=========\nChangelog\n=========\n\n.. _v1_0_a26:\n\n1.0a26 (2026-03-18)\n-------------------\n\nNew ``column_types`` system\n~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nTable columns can now have custom column types assigned to them, using the new ``column_types`` table configuration option or at runtime using a new UI and ``POST ///-/set-column-type`` JSON API.\n\nBuilt-in column types include ``url``, ``email``, and ``json``, and plugins can register additional types using the new :ref:`register_column_types() ` plugin hook. (:issue:`2664`, :issue:`2671`)\n\nColumn types can customize HTML rendering, validate values written through the insert, update, and upsert APIs, and transform values returned by the JSON API. They can optionally restrict themselves to specific SQLite column types using ``sqlite_types``. This feature also introduces a new :ref:`set-column-type ` permission for assigning column types to a table. (:issue:`2672`)\n\nThe :ref:`render_cell() ` plugin hook now receives a ``column_type`` argument containing the assigned type instance, and a column type's own ``render_cell()`` method takes priority over the plugin hook chain.\n\nThe `datasette-files `__ plugin will be the first to use this new feature.\n\nUI for selecting columns and their order\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nTable and view pages now include a dialog for selecting and re-ordering visible columns. (:issue:`2661`)\n\nOther changes\n~~~~~~~~~~~~~\n\n- Fixed ``allowed_resources(\"view-query\", actor)`` so actor-specific canned queries are returned correctly. Any plugin that defines a ``resources_sql()`` method on a ``Resource`` subclass needs to update to the new signature, see :ref:`the resources_sql() method` documentation for details.\n- Column actions can now be accessed in mobile view via a new \"Column actions\" button. Previously they were not available on mobile because table headers are not displayed there. (:issue:`2669`, :issue:`2670`)\n- Row pages now render foreign key values as links to the referenced row. (:issue:`1592`)\n- The ``startup()`` plugin hook now fires after metadata and internal schema tables have been populated, so plugins can reliably inspect that state during startup. (:issue:`2666`)\n\n.. _v1_0_a25:\n\n1.0a25 (2026-02-25)\n-------------------\n\n``write_wrapper()`` plugin hook for intercepting write operations\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nA new :ref:`write_wrapper() ` plugin hook allows plugins to intercept and wrap database write operations. (`#2636 `__)\n\nPlugins implement the hook as a generator-based context manager:\n\n.. code-block:: python\n\n    @hookimpl\n    def write_wrapper(datasette, database, request):\n        def wrapper(conn):\n            # Setup code runs before the write\n            yield\n            # Cleanup code runs after the write\n\n        return wrapper\n\n``register_token_handler()`` plugin hook for custom API token backends\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nA new :ref:`register_token_handler() ` plugin hook allows plugins to provide custom token backends for API authentication. (`#2650 `__)\n\nThis includes a **backwards incompatible change**: the ``datasette.create_token()`` internal  method is now an ``async`` method. Consult the :ref:`upgrade guide ` for details on how to update your code.\n\n``render_cell()`` now receives a ``pks`` parameter\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nThe :ref:`render_cell() ` plugin hook now receives a ``pks`` parameter containing the list of primary key column names for the table being rendered. This avoids plugins needing to make redundant async calls to look up primary keys. (`#2641 `__)\n\nOther changes\n~~~~~~~~~~~~~\n\n- Facets defined in metadata now preserve their configured order, instead of being sorted by result count. Request-based facets added via the ``_facet`` parameter are still sorted by result count and appear after metadata-defined facets. (:issue:`2647`)\n- Fixed ``--reload`` incorrectly interpreting the ``serve`` command as a file argument. Thanks, `Daniel Bates `__. (`#2646 `__)\n\n.. _v1_0_a24:\n\n1.0a24 (2026-01-29)\n-------------------\n\n``request.form()`` method for POST data and file uploads\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nDatasette now includes a ``request.form()`` method for parsing form submissions, including handling file uploads. (`#2626 `__)\n\nThis supports both ``application/x-www-form-urlencoded`` and ``multipart/form-data`` content types, and uses a new streaming multipart parser that processes uploads without buffering entire request bodies in memory.\n\n.. code-block:: python\n\n    # Parse form fields (files are discarded by default)\n    form = await request.form()\n    username = form[\"username\"]\n\n    # Parse form fields AND file uploads\n    form = await request.form(files=True)\n    uploaded = form[\"avatar\"]\n    content = await uploaded.read()\n\nThe returned :ref:`FormData ` object provides dictionary-style access with support for multiple values per key via ``form.getlist(\"key\")``. Uploaded files are represented as :ref:`UploadedFile ` objects with ``filename``, ``content_type``, ``size`` properties and async ``read()`` and ``seek()`` methods.\n\nFiles smaller than 1MB are held in memory; larger files automatically spill to temporary files on disk. Configurable limits control maximum file size, request size, field counts and more.\n\nSeveral internal views (permissions debug, messages debug, create token) now use ``request.form()`` instead of ``request.post_vars()``.\n\n``request.post_vars()`` remains available for backwards compatibility but is no longer the recommended API for handling POST data.\n\n``render_cell`` and ``foreign_key_tables`` extras for the JSON API\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nThe table JSON API now supports ``?_extra=render_cell``, which returns the rendered HTML for each cell as produced by the :ref:`render_cell plugin hook `. Only columns whose rendered output differs from the default are included. (:issue:`2619`)\n\nThe row JSON API also gains ``?_extra=render_cell`` and ``?_extra=foreign_key_tables`` extras, bringing it closer to parity with the table API.\n\nThe row JSON API now returns ``\"ok\": true`` in its response, for consistency with the table API.\n\n``uv run pytest`` with a ``dev=`` dependency group\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nThe recommended development environment for Datasette now uses `uv `__. You can now set up a development environment and run the test suite with just ``uv run pytest`` — no manual virtualenv or ``pip install`` step required. (:issue:`2611`)\n\nOther changes\n~~~~~~~~~~~~~\n\n- Plugins that raise ``datasette.utils.StartupError()`` during startup now display a clean error message instead of a full traceback. (:issue:`2624`)\n- Schema refreshes are now throttled to at most once per second, providing a small performance increase. (:issue:`2629`)\n- Minor performance improvement to ``remove_infinites`` — rows without infinity values now skip the list/dict reconstruction step. (:issue:`2629`)\n- Filter inputs and the search input no longer trigger unwanted zoom on iOS Safari. Thanks, `Daniel Olasubomi Sobowale `__. (:issue:`2346`)\n- ``table_names()`` and ``get_all_foreign_keys()`` now return results in deterministic sorted order. (:issue:`2628`)\n- Switched linting to `ruff `__ and fixed all lint errors. (:issue:`2630`)\n\n.. _v1_0_a23:\n\n1.0a23 (2025-12-02)\n-------------------\n\n- Fix for bug where a stale database entry in ``internal.db`` could cause a 500 error on the homepage. (:issue:`2605`)\n- Cosmetic improvement to ``/-/actions`` page. (:issue:`2599`)\n\n.. _v1_0_a22:\n\n1.0a22 (2025-11-13)\n-------------------\n\n- ``datasette serve --default-deny`` option for running Datasette configured to  :ref:`deny all permissions by default `. (:issue:`2592`)\n- ``datasette.is_client()`` method for detecting if code is :ref:`executing inside a datasette.client request `. (:issue:`2594`)\n- ``datasette.pm`` property can now be used to :ref:`register and unregister plugins in tests `. (:issue:`2595`)\n\n.. _v1_0_a21:\n\n1.0a21 (2025-11-05)\n-------------------\n\n- Fixes an **open redirect** security issue: Datasette instances would redirect to ``example.com/foo/bar`` if you accessed the path ``//example.com/foo/bar``. Thanks to `James Jefferies `__ for the fix. (:issue:`2429`)\n- Fixed ``datasette publish cloudrun`` to work with changes to the underlying Cloud Run architecture. (:issue:`2511`)\n- New ``datasette --get /path --headers`` option for inspecting the headers returned by a path. (:issue:`2578`)\n- New ``datasette.client.get(..., skip_permission_checks=True)`` parameter to bypass permission checks when making requests using the internal client. (:issue:`2583`)\n\n.. _v0_65_2:\n\n0.65.2 (2025-11-05)\n-------------------\n\n- Fixes an **open redirect** security issue: Datasette instances would redirect to ``example.com/foo/bar`` if you accessed the path ``//example.com/foo/bar``. Thanks to `James Jefferies `__ for the fix. (:issue:`2429`)\n- Upgraded for compatibility with Python 3.14.\n- Fixed ``datasette publish cloudrun`` to work with changes to the underlying Cloud Run architecture. (:issue:`2511`)\n- Minor upgrades to fix warnings, including ``pkg_resources`` deprecation.\n\n.. _v1_0_a20:\n\n1.0a20 (2025-11-03)\n-------------------\n\nThis alpha introduces a major breaking change prior to the 1.0 release of Datasette concerning how Datasette's permission system works.\n\nPermission system redesign\n~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nPreviously the permission system worked using ``datasette.permission_allowed()`` checks which consulted all available plugins in turn to determine whether a given actor was allowed to perform a given action on a given resource.\n\nThis approach could become prohibitively expensive for large lists of items - for example to determine the list of tables that a user could view in a large Datasette instance each plugin implementation of that hook would be fired for every table.\n\nThe new design uses SQL queries against Datasette's internal :ref:`catalog tables ` to derive the list of resources for which an actor has permission for a given action. This turns an N x M problem (N resources, M plugins) into a single SQL query.\n\nPlugins can use the new :ref:`plugin_hook_permission_resources_sql` hook to return SQL fragments which will be used as part of that query.\n\nPlugins that use any of the following features will need to be updated to work with this and following alphas (and Datasette 1.0 stable itself):\n\n- Checking permissions with ``datasette.permission_allowed()`` - this method has been replaced with :ref:`datasette.allowed() `.\n- Implementing the ``permission_allowed()`` plugin hook - this hook has been removed in favor of :ref:`permission_resources_sql() `.\n- Using ``register_permissions()`` to register permissions - this hook has been removed in favor of :ref:`register_actions() `.\n\nConsult the :ref:`v1.0a20 upgrade guide ` for further details on how to upgrade affected plugins.\n\nPlugins can now make use of two new internal methods to help resolve permission checks:\n\n- :ref:`datasette.allowed_resources() ` returns a ``PaginatedResources`` object with a ``.resources`` list of ``Resource`` instances that an actor is allowed to access for a given action (and a ``.next`` token for pagination).\n- :ref:`datasette.allowed_resources_sql() ` returns the SQL and parameters that can be executed against the internal catalog tables to determine which resources an actor is allowed to access for a given action. This can be combined with further SQL to perform advanced custom filtering.\n\nRelated changes:\n\n- The way ``datasette --root`` works has changed. Running Datasette with this flag now causes the root actor to pass *all* permission checks. (:issue:`2521`)\n\n- Permission debugging improvements:\n\n  - The ``/-/allowed`` endpoint shows resources the user is allowed to interact with for different actions.\n  - ``/-/rules`` shows the raw allow/deny rules that apply to different permission checks.\n  - ``/-/actions`` lists every available action.\n  - ``/-/check`` can be used to try out different permission checks for the current actor.\n\nOther changes\n~~~~~~~~~~~~~\n\n- The internal ``catalog_views`` table now tracks SQLite views alongside tables in the introspection database. (:issue:`2495`)\n- Hitting the ``/`` brings up a search interface for navigating to tables that the current user can view. A new ``/-/tables`` endpoint supports this functionality. (:issue:`2523`)\n- Datasette attempts to detect some configuration errors on startup.\n- Datasette now supports Python 3.14 and no longer tests against Python 3.9.\n\n.. _v1_0_a19:\n\n1.0a19 (2025-04-21)\n-------------------\n\n- Tiny cosmetic bug fix for mobile display of table rows. (:issue:`2479`)\n\n.. _v1_0_a18:\n\n1.0a18 (2025-04-16)\n-------------------\n\n- Fix for incorrect foreign key references in the internal database schema. (:issue:`2466`)\n- The ``prepare_connection()`` hook no longer runs for the internal database. (:issue:`2468`)\n- Fixed bug where ``link:`` HTTP headers used invalid syntax. (:issue:`2470`)\n- No longer tested against Python 3.8. Now tests against Python 3.13.\n- FTS tables are now hidden by default if they correspond to a content table. (:issue:`2477`)\n- Fixed bug with foreign key links to rows in databases with filenames containing a special character. Thanks, `Jack Stratton `__. (`#2476 `__)\n\n.. _v1_0_a17:\n\n1.0a17 (2025-02-06)\n-------------------\n\n- ``DATASETTE_SSL_KEYFILE`` and ``DATASETTE_SSL_CERTFILE`` environment variables as alternatives to ``--ssl-keyfile`` and ``--ssl-certfile``. Thanks, Alex Garcia. (:issue:`2422`)\n- ``SQLITE_EXTENSIONS`` environment variable has been renamed to ``DATASETTE_LOAD_EXTENSION``. (:issue:`2424`)\n- ``datasette serve`` environment variables are now :ref:`documented here `.\n- The :ref:`plugin_hook_register_magic_parameters` plugin hook can now register async functions. (:issue:`2441`)\n- Datasette is now tested against Python 3.13.\n- Breadcrumbs on database and table pages now include a consistent self-link for resetting query string parameters. (:issue:`2454`)\n- Fixed issue where Datasette could crash on ``metadata.json`` with nested values. (:issue:`2455`)\n- New internal methods ``datasette.set_actor_cookie()`` and ``datasette.delete_actor_cookie()``, :ref:`described here `. (:issue:`1690`)\n- ``/-/permissions`` page now shows a list of all permissions registered by plugins. (:issue:`1943`)\n- If a table has a single unique text column Datasette now detects that as the foreign key label for that table. (:issue:`2458`)\n- The ``/-/permissions`` page now includes options for filtering or exclude permission checks recorded against the current user. (:issue:`2460`)\n- Fixed a bug where replacing a database with a new one with the same name did not pick up the new database correctly. (:issue:`2465`)\n\n.. _v0_65_1:\n\n0.65.1 (2024-11-28)\n-------------------\n\n- Fixed bug with upgraded HTTPX 0.28.0 dependency. (:issue:`2443`)\n\n.. _v0_65:\n\n0.65 (2024-10-07)\n-----------------\n\n- Upgrade for compatibility with Python 3.13 (by vendoring Pint dependency). (:issue:`2434`)\n- Dropped support for Python 3.8.\n\n.. _v1_0_a16:\n\n1.0a16 (2024-09-05)\n-------------------\n\nThis release focuses on performance, in particular against large tables, and introduces some minor breaking changes for CSS styling in Datasette plugins.\n\n- Removed the unit conversions feature and its dependency, Pint. This means Datasette is now compatible with the upcoming Python 3.13. (:issue:`2400`, :issue:`2320`)\n- The ``datasette --pdb`` option now uses the `ipdb `__ debugger if it is installed. You can install it using ``datasette install ipdb``. Thanks, `Tiago Ilieve `__. (`#2342 `__)\n- Fixed a confusing error that occurred if ``metadata.json`` contained nested objects. (:issue:`2403`)\n- Fixed a bug with ``?_trace=1`` where it returned a blank page if the response was larger than 256KB. (:issue:`2404`)\n- Tracing mechanism now also displays SQL queries that returned errors or ran out of time. `datasette-pretty-traces 0.5 `__ includes support for displaying this new type of trace. (:issue:`2405`)\n- Fixed a text spacing with table descriptions on the homepage. (:issue:`2399`)\n- Performance improvements for large tables:\n    - Suggested facets now only consider the first 1000 rows. (:issue:`2406`)\n    - Improved performance of date facet suggestion against large tables. (:issue:`2407`)\n    - Row counts stop at 10,000 rows when listing tables. (:issue:`2398`)\n    - On table page the count stops at 10,000 rows too, with a \"count all\" button to execute the full count. (:issue:`2408`)\n- New ``.dicts()`` internal method on :ref:`database_results` that returns a list of dictionaries representing the results from a SQL query: (:issue:`2414`)\n\n  .. code-block:: bash\n\n        rows = (await db.execute(\"select * from t\")).dicts()\n\n- Default Datasette core CSS that styles inputs and buttons now requires a class of ``\"core\"`` on the element or a containing element, for example ``
    ``. (:issue:`2415`)\n- Similarly, default table styles now only apply to ``
    ``. (:issue:`2420`)\n\n.. _v1_0_a15:\n\n1.0a15 (2024-08-15)\n-------------------\n\n- Datasette now defaults to hiding SQLite \"shadow\" tables, as seen in extensions such as SQLite FTS and `sqlite-vec `__. Virtual tables that it makes sense to display, such as FTS core tables, are no longer hidden. Thanks, `Alex Garcia `__. (:issue:`2296`)\n- Fixed bug where running Datasette with one or more ``-s/--setting`` options could over-ride settings that were present in ``datasette.yml``. (:issue:`2389`)\n- The Datasette homepage is now duplicated at ``/-/``, using the default ``index.html`` template. This ensures that the information on that page is still accessible even if the Datasette homepage has been customized using a custom ``index.html`` template, for example on sites like `datasette.io `__. (:issue:`2393`)\n- Failed CSRF checks now display a more user-friendly error page. (:issue:`2390`)\n- Fixed a bug where the ``json1`` extension was not correctly detected on the ``/-/versions`` page. Thanks, `Seb Bacon `__. (:issue:`2326`)\n- Fixed a bug where the Datasette write API did not correctly accept ``Content-Type: application/json; charset=utf-8``. (:issue:`2384`)\n- Fixed a bug where Datasette would fail to start if ``metadata.yml`` contained a ``queries`` block. (`#2386 `__)\n\n.. _v1_0_a14:\n\n1.0a14 (2024-08-05)\n-------------------\n\nThis alpha introduces significant changes to Datasette's :ref:`metadata` system, some of which represent breaking changes in advance of the full 1.0 release. The new :ref:`upgrade_guide` document provides detailed coverage of those breaking changes and how they affect plugin authors and Datasette API consumers.\n\n- The ``/databasename?sql=`` interface and JSON API for executing arbitrary SQL queries can now be found at ``/databasename/-/query?sql=``. Requests with a ``?sql=`` parameter to the old endpoints will be redirected. Thanks, `Alex Garcia `__. (:issue:`2360`)\n- Metadata about tables, databases, instances and columns is now stored in :ref:`internals_internal`. Thanks, Alex Garcia. (:issue:`2341`)\n- Database write connections now execute using the ``IMMEDIATE`` isolation level for SQLite. This should help avoid a rare ``SQLITE_BUSY`` error that could occur when a transaction upgraded to a write mid-flight. (:issue:`2358`)\n- Fix for a bug where canned queries with named parameters could fail against SQLite 3.46. (:issue:`2353`)\n- Datasette now serves ``E-Tag`` headers for static files. Thanks, `Agustin Bacigalup `__. (`#2306 `__)\n- Dropdown menus now use a ``z-index`` that should avoid them being hidden by plugins. (:issue:`2311`)\n- Incorrect table and row names are no longer reflected back on the resulting 404 page. (:issue:`2359`)\n- Improved documentation for async usage of the :ref:`plugin_hook_track_event` hook. (:issue:`2319`)\n- Fixed some HTTPX deprecation warnings. (:issue:`2307`)\n- Datasette now serves a ```` attribute. Thanks, `Charles Nepote `__. (:issue:`2348`)\n- Datasette's automated tests now run against the maximum and minimum supported versions of SQLite: 3.25 (from September 2018) and 3.46 (from May 2024). Thanks, Alex Garcia. (`#2352 `__)\n- Fixed an issue where clicking twice on the URL output by ``datasette --root`` produced a confusing error. (:issue:`2375`)\n\n.. _v0_64_8:\n\n0.64.8 (2024-06-21)\n-------------------\n\n- Security improvement: 404 pages used to reflect content from the URL path, which could be used to display misleading information to Datasette users. 404 errors no longer display additional information from the URL. (:issue:`2359`)\n- Backported a better fix for correctly extracting named parameters from canned query SQL against SQLite 3.46.0. (:issue:`2353`)\n\n.. _v0_64_7:\n\n0.64.7 (2024-06-12)\n-------------------\n\n- Fixed a bug where canned queries with named parameters threw an error when run against SQLite 3.46.0. (:issue:`2353`)\n\n.. _v1_0_a13:\n\n1.0a13 (2024-03-12)\n-------------------\n\nEach of the key concepts in Datasette now has an :ref:`actions menu `, which plugins can use to add additional functionality targeting that entity.\n\n- Plugin hook: :ref:`view_actions() ` for actions that can be applied to a SQL view. (:issue:`2297`)\n- Plugin hook: :ref:`homepage_actions() ` for actions that apply to the instance homepage. (:issue:`2298`)\n- Plugin hook: :ref:`row_actions() ` for actions that apply to the row page. (:issue:`2299`)\n- Action menu items for all of the ``*_actions()`` plugin hooks can now return an optional ``\"description\"`` key, which will be displayed in the menu below the action label. (:issue:`2294`)\n- :ref:`Plugin hooks ` documentation page is now organized with additional headings. (:issue:`2300`)\n- Improved the display of action buttons on pages that also display metadata. (:issue:`2286`)\n- The header and footer of the page now uses a subtle gradient effect, and options in the navigation menu are better visually defined. (:issue:`2302`)\n- Table names that start with an underscore now default to hidden. (:issue:`2104`)\n- ``pragma_table_list`` has been added to the allow-list of SQLite pragma functions supported by Datasette. ``select * from pragma_table_list()`` is no longer blocked. (`#2104 `__)\n\n.. _v1_0_a12:\n\n1.0a12 (2024-02-29)\n-------------------\n\n- New :ref:`query_actions() ` plugin hook, similar to :ref:`table_actions() ` and :ref:`database_actions() `. Can be used to add a menu of actions to the canned query or arbitrary SQL query page. (:issue:`2283`)\n- New design for the button that opens the query, table and database actions menu. (:issue:`2281`)\n- \"does not contain\" table filter for finding rows that do not contain a string. (:issue:`2287`)\n- Fixed a bug in the :ref:`javascript_plugins_makeColumnActions` JavaScript plugin mechanism where the column action menu was not fully reset in between each interaction. (:issue:`2289`)\n\n.. _v1_0_a11:\n\n1.0a11 (2024-02-19)\n-------------------\n\n- The ``\"replace\": true`` argument to the ``/db/table/-/insert`` API now requires the actor to have the ``update-row`` permission. (:issue:`2279`)\n- Fixed some UI bugs in the interactive permissions debugging tool. (:issue:`2278`)\n- The column action menu now aligns better with the cog icon, and positions itself taking into account the width of the browser window. (:issue:`2263`)\n\n.. _v1_0_a10:\n\n1.0a10 (2024-02-17)\n-------------------\n\nThe only changes in this alpha correspond to the way Datasette handles database transactions. (:issue:`2277`)\n\n- The :ref:`database.execute_write_fn() ` method has a new ``transaction=True`` parameter. This defaults to ``True`` which means all functions executed using this method are now automatically wrapped in a transaction - previously the functions needed to roll transaction handling on their own, and many did not.\n- Pass ``transaction=False`` to ``execute_write_fn()`` if you want to manually handle transactions in your function.\n- Several internal Datasette features, including parts of the :ref:`JSON write API `, had been failing to wrap their operations in a transaction. This has been fixed by the new ``transaction=True`` default.\n\n.. _v1_0_a9:\n\n1.0a9 (2024-02-16)\n------------------\n\nThis alpha release adds basic alter table support to the Datasette Write API and fixes a permissions bug relating to the ``/upsert`` API endpoint.\n\nAlter table support for create, insert, upsert and update\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nThe :ref:`JSON write API ` can now be used to apply simple alter table schema changes, provided the acting actor has the new :ref:`actions_alter_table` permission. (:issue:`2101`)\n\nThe only alter operation supported so far is adding new columns to an existing table.\n\n* The :ref:`/db/-/create ` API now adds new columns during large operations to create a table based on incoming example ``\"rows\"``, in the case where one of the later rows includes columns that were not present in the earlier batches. This requires the ``create-table`` but not the ``alter-table`` permission.\n* When ``/db/-/create`` is called with rows in a situation where the table may have been already created, an ``\"alter\": true`` key can be included to indicate that any missing columns from the new rows should be added to the table. This requires the ``alter-table`` permission.\n* :ref:`/db/table/-/insert ` and :ref:`/db/table/-/upsert ` and :ref:`/db/table/row-pks/-/update ` all now also accept ``\"alter\": true``, depending on the ``alter-table`` permission.\n\nOperations that alter a table now fire the new :ref:`alter-table event `.\n\nPermissions fix for the upsert API\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nThe :ref:`/database/table/-/upsert API ` had a minor permissions bug, only affecting Datasette instances that had configured the ``insert-row`` and ``update-row`` permissions to apply to a specific table rather than the database or instance as a whole. Full details in issue :issue:`2262`.\n\nTo avoid similar mistakes in the future the ``datasette.permission_allowed()`` method now specifies ``default=`` as a keyword-only argument.\n\nPermission checks now consider opinions from every plugin\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n\nThe ``datasette.permission_allowed()`` method previously consulted every plugin that implemented the ``permission_allowed()`` plugin hook and obeyed the opinion of the last plugin to return a value. (:issue:`2275`)\n\nDatasette now consults every plugin and checks to see if any of them returned ``False`` (the veto rule), and if none of them did, it then checks to see if any of them returned ``True``.\n\nThis is explained at length in the new documentation covering :ref:`authentication_permissions_explained`.\n\nOther changes\n~~~~~~~~~~~~~\n\n- The new :ref:`DATASETTE_TRACE_PLUGINS=1 environment variable ` turns on detailed trace output for every executed plugin hook, useful for debugging and understanding how the plugin system works at a low level. (:issue:`2274`)\n- Datasette on Python 3.9 or above marks its non-cryptographic uses of the MD5 hash function as ``usedforsecurity=False``, for compatibility with FIPS systems. (:issue:`2270`)\n- SQL relating to :ref:`internals_internal` now executes inside a transaction, avoiding a potential database locked error. (:issue:`2273`)\n- The ``/-/threads`` debug page now identifies the database in the name associated with each dedicated write thread. (:issue:`2265`)\n- The ``/db/-/create`` API now fires a ``insert-rows`` event if rows were inserted after the table was created. (:issue:`2260`)\n\n.. _v1_0_a8:\n\n1.0a8 (2024-02-07)\n------------------\n\nThis alpha release continues the migration of Datasette's configuration from ``metadata.yaml`` to the new ``datasette.yaml`` configuration file, introduces a new system for JavaScript plugins and adds several new plugin hooks.\n\nSee `Datasette 1.0a8: JavaScript plugins, new plugin hooks and plugin configuration in datasette.yaml `__ for an annotated version of these release notes.\n\nConfiguration\n~~~~~~~~~~~~~\n\n- Plugin configuration now lives in the :ref:`datasette.yaml configuration file `, passed to Datasette using the ``-c/--config`` option. Thanks, Alex Garcia. (:issue:`2093`)\n\n  .. code-block:: bash\n\n        datasette -c datasette.yaml\n\n  Where ``datasette.yaml`` contains configuration that looks like this:\n\n  .. code-block:: yaml\n\n        plugins:\n          datasette-cluster-map:\n            latitude_column: xlat\n            longitude_column: xlon\n\n  Previously plugins were configured in ``metadata.yaml``, which was confusing as plugin settings were unrelated to database and table metadata.\n- The ``-s/--setting`` option can now be used to set plugin configuration as well. See :ref:`configuration_cli` for details. (:issue:`2252`)\n\n  The above YAML configuration example using ``-s/--setting`` looks like this:\n\n  .. code-block:: bash\n\n        datasette mydatabase.db \\\n          -s plugins.datasette-cluster-map.latitude_column xlat \\\n          -s plugins.datasette-cluster-map.longitude_column xlon\n\n- The new ``/-/config`` page shows the current instance configuration, after redacting keys that could contain sensitive data such as API keys or passwords. (:issue:`2254`)\n\n- Existing Datasette installations may already have configuration set in ``metadata.yaml`` that should be migrated to ``datasette.yaml``. To avoid breaking these installations, Datasette will silently treat table configuration, plugin configuration and allow blocks in metadata as if they had been specified in configuration instead. (:issue:`2247`) (:issue:`2248`) (:issue:`2249`)\n\nNote that the ``datasette publish`` command has not yet been updated to accept a ``datasette.yaml`` configuration file. This will be addressed in :issue:`2195` but for the moment you can include those settings in ``metadata.yaml`` instead.\n\nJavaScript plugins\n~~~~~~~~~~~~~~~~~~\n\nDatasette now includes a :ref:`JavaScript plugins mechanism `, allowing JavaScript to customize Datasette in a way that can collaborate with other plugins.\n\nThis provides two initial hooks, with more to come in the future:\n\n- :ref:`makeAboveTablePanelConfigs() ` can add additional panels to the top of the table page.\n- :ref:`makeColumnActions() ` can add additional actions to the column menu.\n\nThanks `Cameron Yick `__ for contributing this feature. (`#2052 `__)\n\nPlugin hooks\n~~~~~~~~~~~~\n\n- New :ref:`plugin_hook_jinja2_environment_from_request` plugin hook, which can be used to customize the current Jinja environment based on the incoming request. This can be used to modify the template lookup path based on the incoming request hostname, among other things. (:issue:`2225`)\n- New :ref:`family of template slot plugin hooks `: ``top_homepage``, ``top_database``, ``top_table``, ``top_row``, ``top_query``, ``top_canned_query``. Plugins can use these to provide additional HTML to be injected at the top of the corresponding pages. (:issue:`1191`)\n- New :ref:`track_event() mechanism ` for plugins to emit and receive events when certain events occur within Datasette. (:issue:`2240`)\n    - Plugins can register additional event classes using :ref:`plugin_hook_register_events`.\n    - They can then trigger those events with the :ref:`datasette.track_event(event) ` internal method.\n    - Plugins can subscribe to notifications of events using the :ref:`plugin_hook_track_event` plugin hook.\n    - Datasette core now emits ``login``, ``logout``, ``create-token``, ``create-table``, ``drop-table``, ``insert-rows``, ``upsert-rows``, ``update-row``, ``delete-row`` events, :ref:`documented here `.\n- New internal function for plugin authors: :ref:`database_execute_isolated_fn`, for creating a new SQLite connection, executing code and then closing that connection, all while preventing other code from writing to that particular database. This connection will not have the :ref:`prepare_connection() ` plugin hook executed against it, allowing plugins to perform actions that might otherwise be blocked by existing connection configuration. (:issue:`2218`)\n\nDocumentation\n~~~~~~~~~~~~~\n\n- Documentation describing :ref:`how to write tests that use signed actor cookies ` using ``datasette.client.actor_cookie()``. (:issue:`1830`)\n- Documentation on how to :ref:`register a plugin for the duration of a test `. (:issue:`2234`)\n- The :ref:`configuration documentation ` now shows examples of both YAML and JSON for each setting.\n\nMinor fixes\n~~~~~~~~~~~\n\n- Datasette no longer attempts to run SQL queries in parallel when rendering a table page, as this was leading to some rare crashing bugs. (:issue:`2189`)\n- Fixed warning: ``DeprecationWarning: pkg_resources is deprecated as an API`` (:issue:`2057`)\n- Fixed bug where ``?_extra=columns`` parameter returned an incorrectly shaped response. (:issue:`2230`)\n\n.. _v0_64_6:\n\n0.64.6 (2023-12-22)\n-------------------\n\n- Fixed a bug where CSV export with expanded labels could fail if a foreign key reference did not correctly resolve. (:issue:`2214`)\n\n.. _v0_64_5:\n\n0.64.5 (2023-10-08)\n-------------------\n\n- Dropped dependency on ``click-default-group-wheel``, which could cause a dependency conflict. (:issue:`2197`)\n\n.. _v1_0_a7:\n\n1.0a7 (2023-09-21)\n------------------\n\n- Fix for a crashing bug caused by viewing the table page for a named in-memory database. (:issue:`2189`)\n\n.. _v0_64_4:\n\n0.64.4 (2023-09-21)\n-------------------\n\n- Fix for a crashing bug caused by viewing the table page for a named in-memory database. (:issue:`2189`)\n\n.. _v1_0_a6:\n\n1.0a6 (2023-09-07)\n------------------\n\n- New plugin hook: :ref:`plugin_hook_actors_from_ids` and an internal method to accompany it, :ref:`datasette_actors_from_ids`. This mechanism is intended to be used by plugins that may need to display the actor who was responsible for something managed by that plugin: they can now resolve the recorded IDs of actors into the full actor objects. (:issue:`2181`)\n- ``DATASETTE_LOAD_PLUGINS`` environment variable for :ref:`controlling which plugins ` are loaded by Datasette. (:issue:`2164`)\n- Datasette now checks if the user has permission to view a table linked to by a foreign key before turning that foreign key into a clickable link. (:issue:`2178`)\n- The ``execute-sql`` permission now implies that the actor can also view the database and instance. (:issue:`2169`)\n- Documentation describing a pattern for building plugins that themselves :ref:`define further hooks ` for other plugins. (:issue:`1765`)\n- Datasette is now tested against the Python 3.12 preview. (`#2175 `__)\n\n.. _v1_0_a5:\n\n1.0a5 (2023-08-29)\n------------------\n\n- When restrictions are applied to :ref:`API tokens `, those restrictions now behave slightly differently: applying the ``view-table`` restriction will imply the ability to ``view-database`` for the database containing that table, and both ``view-table`` and ``view-database`` will imply ``view-instance``. Previously you needed to create a token with restrictions that explicitly listed ``view-instance`` and ``view-database`` and ``view-table`` in order to view a table without getting a permission denied error. (:issue:`2102`)\n- New ``datasette.yaml`` (or ``.json``) configuration file, which can be specified using ``datasette -c path-to-file``. The goal here to consolidate settings, plugin configuration, permissions, canned queries, and other Datasette configuration into a single single file, separate from ``metadata.yaml``. The legacy ``settings.json`` config file used for :ref:`config_dir` has been removed, and ``datasette.yaml`` has a ``\"settings\"`` section where the same settings key/value pairs can be included. In the next future alpha release, more configuration such as plugins/permissions/canned queries will be moved to the ``datasette.yaml`` file. See :issue:`2093` for more details. Thanks, Alex Garcia.\n- The ``-s/--setting`` option can now take dotted paths to nested settings. These will then be used to set or over-ride the same options as are present in the new configuration file. (:issue:`2156`)\n- New ``--actor '{\"id\": \"json-goes-here\"}'`` option for use with ``datasette --get`` to treat the simulated request as being made by a specific actor, see :ref:`cli_datasette_get`. (:issue:`2153`)\n- The Datasette ``_internal`` database has had some changes. It no longer shows up in the ``datasette.databases`` list by default, and is now instead available to plugins using the ``datasette.get_internal_database()``. Plugins are invited to use this as a private database to store configuration and settings and secrets that should not be made visible through the default Datasette interface. Users can pass the new  ``--internal internal.db`` option to persist that internal database to disk. Thanks, Alex Garcia. (:issue:`2157`).\n\n.. _v1_0_a4:\n\n1.0a4 (2023-08-21)\n------------------\n\nThis alpha fixes a security issue with the ``/-/api`` API explorer. On authenticated Datasette instances (instances protected using plugins such as `datasette-auth-passwords `__) the API explorer interface could reveal the names of databases and tables within the protected instance. The data stored in those tables was not revealed.\n\nFor more information and workarounds, read `the security advisory `__. The issue has been present in every previous alpha version of Datasette 1.0: versions 1.0a0, 1.0a1, 1.0a2 and 1.0a3.\n\nAlso in this alpha:\n\n- The new ``datasette plugins --requirements`` option outputs a list of currently installed plugins in Python ``requirements.txt`` format, useful for duplicating that installation elsewhere. (:issue:`2133`)\n- :ref:`canned_queries_writable` can now define a ``on_success_message_sql`` field in their configuration, containing a SQL query that should be executed upon successful completion of the write operation in order to generate a message to be shown to the user. (:issue:`2138`)\n- The automatically generated border color for a database is now shown in more places around the application. (:issue:`2119`)\n- Every instance of example shell script code in the documentation should now include a working copy button, free from additional syntax. (:issue:`2140`)\n\n.. _v1_0_a3:\n\n1.0a3 (2023-08-09)\n------------------\n\nThis alpha release previews the updated design for Datasette's default JSON API. (:issue:`782`)\n\nThe new :ref:`default JSON representation ` for both table pages (``/dbname/table.json``) and arbitrary SQL queries (``/dbname.json?sql=...``) is now shaped like this:\n\n.. code-block:: json\n\n    {\n      \"ok\": true,\n      \"rows\": [\n        {\n          \"id\": 3,\n          \"name\": \"Detroit\"\n        },\n        {\n          \"id\": 2,\n          \"name\": \"Los Angeles\"\n        },\n        {\n          \"id\": 4,\n          \"name\": \"Memnonia\"\n        },\n        {\n          \"id\": 1,\n          \"name\": \"San Francisco\"\n        }\n      ],\n      \"truncated\": false\n    }\n\nTables will include an additional ``\"next\"`` key for pagination, which can be passed to ``?_next=`` to fetch the next page of results.\n\nThe various ``?_shape=`` options continue to work as before - see :ref:`json_api_shapes` for details.\n\nA new ``?_extra=`` mechanism is available for tables, but has not yet been stabilized or documented. Details on that are available in :issue:`262`.\n\nSmaller changes\n~~~~~~~~~~~~~~~\n\n- Datasette documentation now shows YAML examples for :ref:`metadata` by default, with a tab interface for switching to JSON. (:issue:`1153`)\n- :ref:`plugin_register_output_renderer` plugins now have access to ``error`` and ``truncated`` arguments, allowing them to display error messages and take into account truncated results. (:issue:`2130`)\n- ``render_cell()`` plugin hook now also supports an optional ``request`` argument. (:issue:`2007`)\n- New ``Justfile`` to support development workflows for Datasette using `Just `__.\n- ``datasette.render_template()`` can now accepts a ``datasette.views.Context`` subclass as an alternative to a dictionary. (:issue:`2127`)\n- ``datasette install -e path`` option for editable installations, useful while developing plugins. (:issue:`2106`)\n- When started with the ``--cors`` option Datasette now serves an ``Access-Control-Max-Age: 3600`` header, ensuring CORS OPTIONS requests are repeated no more than once an hour. (:issue:`2079`)\n- Fixed a bug where the ``_internal`` database could display ``None`` instead of ``null`` for in-memory databases. (:issue:`1970`)\n\n.. _v0_64_2:\n\n0.64.2 (2023-03-08)\n-------------------\n\n- Fixed a bug with ``datasette publish cloudrun`` where deploys all used the same Docker image tag. This was mostly inconsequential as the service is deployed as soon as the image has been pushed to the registry, but could result in the incorrect image being deployed if two different deploys for two separate services ran at exactly the same time. (:issue:`2036`)\n\n.. _v0_64_1:\n\n0.64.1 (2023-01-11)\n-------------------\n\n- Documentation now links to a current source of information for installing Python 3. (:issue:`1987`)\n- Incorrectly calling the Datasette constructor using ``Datasette(\"path/to/data.db\")`` instead of ``Datasette([\"path/to/data.db\"])`` now returns a useful error message. (:issue:`1985`)\n\n.. _v0_64:\n\n0.64 (2023-01-09)\n-----------------\n\n- Datasette now **strongly recommends against allowing arbitrary SQL queries if you are using SpatiaLite**. SpatiaLite includes SQL functions that could cause the Datasette server to crash. See :ref:`spatialite` for more details.\n- New :ref:`setting_default_allow_sql` setting, providing an easier way to disable all arbitrary SQL execution by end users: ``datasette --setting default_allow_sql off``. See also :ref:`authentication_permissions_execute_sql`. (:issue:`1409`)\n- `Building a location to time zone API with SpatiaLite `__ is a new Datasette tutorial showing how to safely use SpatiaLite to create a location to time zone API.\n- New documentation about :ref:`how to debug problems loading SQLite extensions `. The error message shown when an extension cannot be loaded has also been improved. (:issue:`1979`)\n- Fixed an accessibility issue: the ``\\n'\n            \"\"\n        ),\n        \"status\": 400,\n        \"title\": \"SQL Interrupted\",\n    }\n\n\n@pytest.mark.asyncio\nasync def test_custom_sql_time_limit(ds_client):\n    response = await ds_client.get(\n        \"/fixtures/-/query.json?sql=select+sleep(0.01)\",\n    )\n    assert response.status_code == 200\n    response = await ds_client.get(\n        \"/fixtures/-/query.json?sql=select+sleep(0.01)&_timelimit=5\",\n    )\n    assert response.status_code == 400\n    assert response.json()[\"title\"] == \"SQL Interrupted\"\n\n\n@pytest.mark.asyncio\nasync def test_invalid_custom_sql(ds_client):\n    response = await ds_client.get(\n        \"/fixtures/-/query.json?sql=.schema\",\n    )\n    assert response.status_code == 400\n    assert response.json()[\"ok\"] is False\n    assert \"Statement must be a SELECT\" == response.json()[\"error\"]\n\n\n@pytest.mark.asyncio\nasync def test_row(ds_client):\n    response = await ds_client.get(\"/fixtures/simple_primary_key/1.json?_shape=objects\")\n    assert response.status_code == 200\n    assert response.json()[\"ok\"] is True\n    assert response.json()[\"rows\"] == [{\"id\": 1, \"content\": \"hello\"}]\n\n\n@pytest.mark.asyncio\nasync def test_row_strange_table_name(ds_client):\n    response = await ds_client.get(\n        \"/fixtures/table~2Fwith~2Fslashes~2Ecsv/3.json?_shape=objects\"\n    )\n    assert response.status_code == 200\n    assert response.json()[\"rows\"] == [{\"pk\": \"3\", \"content\": \"hey\"}]\n\n\n@pytest.mark.asyncio\nasync def test_row_foreign_key_tables(ds_client):\n    response = await ds_client.get(\n        \"/fixtures/simple_primary_key/1.json?_extras=foreign_key_tables\"\n    )\n    assert response.status_code == 200\n    # Foreign keys are sorted by (other_table, column, other_column)\n    assert response.json()[\"foreign_key_tables\"] == [\n        {\n            \"other_table\": \"complex_foreign_keys\",\n            \"column\": \"id\",\n            \"other_column\": \"f1\",\n            \"count\": 1,\n            \"link\": \"/fixtures/complex_foreign_keys?f1=1\",\n        },\n        {\n            \"other_table\": \"complex_foreign_keys\",\n            \"column\": \"id\",\n            \"other_column\": \"f2\",\n            \"count\": 0,\n            \"link\": \"/fixtures/complex_foreign_keys?f2=1\",\n        },\n        {\n            \"other_table\": \"complex_foreign_keys\",\n            \"column\": \"id\",\n            \"other_column\": \"f3\",\n            \"count\": 1,\n            \"link\": \"/fixtures/complex_foreign_keys?f3=1\",\n        },\n        {\n            \"other_table\": \"foreign_key_references\",\n            \"column\": \"id\",\n            \"other_column\": \"foreign_key_with_blank_label\",\n            \"count\": 0,\n            \"link\": \"/fixtures/foreign_key_references?foreign_key_with_blank_label=1\",\n        },\n        {\n            \"other_table\": \"foreign_key_references\",\n            \"column\": \"id\",\n            \"other_column\": \"foreign_key_with_label\",\n            \"count\": 1,\n            \"link\": \"/fixtures/foreign_key_references?foreign_key_with_label=1\",\n        },\n    ]\n\n\n@pytest.mark.asyncio\nasync def test_row_extra_render_cell():\n    \"\"\"Test that _extra=render_cell returns rendered HTML from render_cell plugin hook on row pages\"\"\"\n    from datasette import hookimpl\n    from datasette.app import Datasette\n\n    class TestRenderCellPlugin:\n        __name__ = \"TestRenderCellPlugin\"\n\n        @hookimpl\n        def render_cell(self, value, column, table, database):\n            # Only modify cells in our test table\n            if table == \"test_render\" and column == \"name\":\n                return f\"{value}\"\n            return None\n\n    ds = Datasette(memory=True)\n    await ds.invoke_startup()\n    db = ds.add_memory_database(\"test_row_render\")\n    await db.execute_write(\n        \"create table test_render (id integer primary key, name text)\"\n    )\n    await db.execute_write(\"insert into test_render values (1, 'Alice')\")\n\n    # Register our test plugin\n    ds.pm.register(TestRenderCellPlugin(), name=\"TestRenderCellPlugin\")\n\n    try:\n        # Request row with _extra=render_cell\n        response = await ds.client.get(\n            \"/test_row_render/test_render/1.json?_extra=render_cell\"\n        )\n        assert response.status_code == 200\n        data = response.json()\n\n        # Verify the response structure\n        assert \"render_cell\" in data\n        assert \"rows\" in data\n\n        # render_cell should be a list with one row (since this is a row page)\n        # Only columns modified by plugins are included (sparse output)\n        render_cell = data[\"render_cell\"]\n        assert len(render_cell) == 1\n\n        # The row: id=1, name='Alice'\n        # The 'name' column should be rendered by our plugin as Alice\n        assert render_cell[0][\"name\"] == \"Alice\"\n        # The 'id' column is not included since no plugin modified it\n        assert \"id\" not in render_cell[0]\n\n        # The regular rows should still contain raw values\n        assert data[\"rows\"] == [{\"id\": 1, \"name\": \"Alice\"}]\n\n    finally:\n        ds.pm.unregister(name=\"TestRenderCellPlugin\")\n\n\ndef test_databases_json(app_client_two_attached_databases_one_immutable):\n    response = app_client_two_attached_databases_one_immutable.get(\"/-/databases.json\")\n    databases = response.json\n    assert 2 == len(databases)\n    extra_database, fixtures_database = databases\n    assert \"extra database\" == extra_database[\"name\"]\n    assert extra_database[\"hash\"] is None\n    assert extra_database[\"is_mutable\"] is True\n    assert extra_database[\"is_memory\"] is False\n\n    assert \"fixtures\" == fixtures_database[\"name\"]\n    assert fixtures_database[\"hash\"] is not None\n    assert fixtures_database[\"is_mutable\"] is False\n    assert fixtures_database[\"is_memory\"] is False\n\n\n@pytest.mark.asyncio\nasync def test_threads_json(ds_client):\n    response = await ds_client.get(\"/-/threads.json\")\n    expected_keys = {\"threads\", \"num_threads\"}\n    if sys.version_info >= (3, 7, 0):\n        expected_keys.update({\"tasks\", \"num_tasks\"})\n    data = response.json()\n    assert set(data.keys()) == expected_keys\n    # Should be at least one _execute_writes thread for __INTERNAL__\n    thread_names = [thread[\"name\"] for thread in data[\"threads\"]]\n    assert \"_execute_writes for database __INTERNAL__\" in thread_names\n\n\n@pytest.mark.asyncio\nasync def test_plugins_json(ds_client):\n    response = await ds_client.get(\"/-/plugins.json\")\n    # Filter out TrackEventPlugin\n    actual_plugins = sorted(\n        [p for p in response.json() if p[\"name\"] != \"TrackEventPlugin\"],\n        key=lambda p: p[\"name\"],\n    )\n    assert EXPECTED_PLUGINS == actual_plugins\n    # Try with ?all=1\n    response = await ds_client.get(\"/-/plugins.json?all=1\")\n    names = {p[\"name\"] for p in response.json()}\n    assert names.issuperset(p[\"name\"] for p in EXPECTED_PLUGINS)\n    assert names.issuperset(DEFAULT_PLUGINS)\n\n\n@pytest.mark.asyncio\nasync def test_versions_json(ds_client):\n    response = await ds_client.get(\"/-/versions.json\")\n    data = response.json()\n    assert \"python\" in data\n    assert \"3.0\" == data.get(\"asgi\")\n    assert \"version\" in data[\"python\"]\n    assert \"full\" in data[\"python\"]\n    assert \"datasette\" in data\n    assert \"version\" in data[\"datasette\"]\n    assert data[\"datasette\"][\"version\"] == __version__\n    assert \"sqlite\" in data\n    assert \"version\" in data[\"sqlite\"]\n    assert \"fts_versions\" in data[\"sqlite\"]\n    assert \"compile_options\" in data[\"sqlite\"]\n    # By default, the json1 extension is enabled in the SQLite\n    # provided by the `ubuntu-latest` github actions runner, and\n    # all versions of SQLite from 3.38.0 onwards\n    assert data[\"sqlite\"][\"extensions\"][\"json1\"]\n\n\n@pytest.mark.asyncio\nasync def test_actions_json(ds_client):\n    original_root_enabled = ds_client.ds.root_enabled\n    try:\n        ds_client.ds.root_enabled = True\n        cookies = {\"ds_actor\": ds_client.actor_cookie({\"id\": \"root\"})}\n        response = await ds_client.get(\"/-/actions.json\", cookies=cookies)\n        data = response.json()\n    finally:\n        ds_client.ds.root_enabled = original_root_enabled\n    assert isinstance(data, list)\n    assert len(data) > 0\n    # Check structure of first action\n    action = data[0]\n    for key in (\n        \"name\",\n        \"abbr\",\n        \"description\",\n        \"takes_parent\",\n        \"takes_child\",\n        \"resource_class\",\n        \"also_requires\",\n    ):\n        assert key in action\n    # Check that some expected actions exist\n    action_names = {a[\"name\"] for a in data}\n    for expected_action in (\n        \"view-instance\",\n        \"view-database\",\n        \"view-table\",\n        \"execute-sql\",\n    ):\n        assert expected_action in action_names\n\n\n@pytest.mark.asyncio\nasync def test_settings_json(ds_client):\n    response = await ds_client.get(\"/-/settings.json\")\n    assert response.json() == {\n        \"default_page_size\": 50,\n        \"default_facet_size\": 30,\n        \"default_allow_sql\": True,\n        \"facet_suggest_time_limit_ms\": 200,\n        \"facet_time_limit_ms\": 200,\n        \"max_returned_rows\": 100,\n        \"max_insert_rows\": 100,\n        \"sql_time_limit_ms\": 200,\n        \"allow_download\": True,\n        \"allow_signed_tokens\": True,\n        \"max_signed_tokens_ttl\": 0,\n        \"allow_facet\": True,\n        \"suggest_facets\": True,\n        \"default_cache_ttl\": 5,\n        \"num_sql_threads\": 1,\n        \"cache_size_kb\": 0,\n        \"allow_csv_stream\": True,\n        \"max_csv_mb\": 100,\n        \"truncate_cells_html\": 2048,\n        \"force_https_urls\": False,\n        \"template_debug\": False,\n        \"trace_debug\": False,\n        \"base_url\": \"/\",\n    }\n\n\ntest_json_columns_default_expected = [\n    {\"intval\": 1, \"strval\": \"s\", \"floatval\": 0.5, \"jsonval\": '{\"foo\": \"bar\"}'}\n]\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"extra_args,expected\",\n    [\n        (\"\", test_json_columns_default_expected),\n        (\"&_json=intval\", test_json_columns_default_expected),\n        (\"&_json=strval\", test_json_columns_default_expected),\n        (\"&_json=floatval\", test_json_columns_default_expected),\n        (\n            \"&_json=jsonval\",\n            [{\"intval\": 1, \"strval\": \"s\", \"floatval\": 0.5, \"jsonval\": {\"foo\": \"bar\"}}],\n        ),\n    ],\n)\nasync def test_json_columns(ds_client, extra_args, expected):\n    sql = \"\"\"\n        select 1 as intval, \"s\" as strval, 0.5 as floatval,\n        '{\"foo\": \"bar\"}' as jsonval\n    \"\"\"\n    path = \"/fixtures/-/query.json?\" + urllib.parse.urlencode(\n        {\"sql\": sql, \"_shape\": \"array\"}\n    )\n    path += extra_args\n    response = await ds_client.get(\n        path,\n    )\n    assert response.json() == expected\n\n\ndef test_config_cache_size(app_client_larger_cache_size):\n    response = app_client_larger_cache_size.get(\"/fixtures/pragma_cache_size.json\")\n    assert response.json[\"rows\"] == [{\"cache_size\": -2500}]\n\n\ndef test_config_force_https_urls():\n    with make_app_client(settings={\"force_https_urls\": True}) as client:\n        response = client.get(\n            \"/fixtures/facetable.json?_size=3&_facet=state&_extra=next_url,suggested_facets\"\n        )\n        assert response.json[\"next_url\"].startswith(\"https://\")\n        assert response.json[\"facet_results\"][\"results\"][\"state\"][\"results\"][0][\n            \"toggle_url\"\n        ].startswith(\"https://\")\n        assert response.json[\"suggested_facets\"][0][\"toggle_url\"].startswith(\"https://\")\n        # Also confirm that request.url and request.scheme are set correctly\n        response = client.get(\"/\")\n        assert client.ds._last_request.url.startswith(\"https://\")\n        assert client.ds._last_request.scheme == \"https\"\n\n\n@pytest.mark.parametrize(\n    \"path,status_code\",\n    [\n        (\"/fixtures.db\", 200),\n        (\"/fixtures.json\", 200),\n        (\"/fixtures/no_primary_key.json\", 200),\n        # A 400 invalid SQL query should still have the header:\n        (\"/fixtures/-/query.json?sql=select+blah\", 400),\n        # Write APIs\n        (\"/fixtures/-/create\", 405),\n        (\"/fixtures/facetable/-/insert\", 405),\n        (\"/fixtures/facetable/-/drop\", 405),\n    ],\n)\ndef test_cors(\n    app_client_with_cors,\n    app_client_two_attached_databases_one_immutable,\n    path,\n    status_code,\n):\n    response = app_client_with_cors.get(\n        path,\n    )\n    assert response.status == status_code\n    assert response.headers[\"Access-Control-Allow-Origin\"] == \"*\"\n    assert (\n        response.headers[\"Access-Control-Allow-Headers\"]\n        == \"Authorization, Content-Type\"\n    )\n    assert response.headers[\"Access-Control-Expose-Headers\"] == \"Link\"\n    assert (\n        response.headers[\"Access-Control-Allow-Methods\"] == \"GET, POST, HEAD, OPTIONS\"\n    )\n    assert response.headers[\"Access-Control-Max-Age\"] == \"3600\"\n    # Same request to app_client_two_attached_databases_one_immutable\n    # should not have those headers - I'm using that fixture because\n    # regular app_client doesn't have immutable fixtures.db which means\n    # the test for /fixtures.db returns a 403 error\n    response = app_client_two_attached_databases_one_immutable.get(\n        path,\n    )\n    assert response.status == status_code\n    assert \"Access-Control-Allow-Origin\" not in response.headers\n    assert \"Access-Control-Allow-Headers\" not in response.headers\n    assert \"Access-Control-Expose-Headers\" not in response.headers\n    assert \"Access-Control-Allow-Methods\" not in response.headers\n    assert \"Access-Control-Max-Age\" not in response.headers\n\n\n@pytest.mark.parametrize(\n    \"path\",\n    (\n        \"/\",\n        \".json\",\n        \"/searchable\",\n        \"/searchable.json\",\n        \"/searchable_view\",\n        \"/searchable_view.json\",\n    ),\n)\ndef test_database_with_space_in_name(app_client_two_attached_databases, path):\n    response = app_client_two_attached_databases.get(\n        \"/extra~20database\" + path, follow_redirects=True\n    )\n    assert response.status == 200\n\n\ndef test_common_prefix_database_names(app_client_conflicting_database_names):\n    # https://github.com/simonw/datasette/issues/597\n    assert [\"foo-bar\", \"foo\", \"fixtures\"] == [\n        d[\"name\"]\n        for d in app_client_conflicting_database_names.get(\"/-/databases.json\").json\n    ]\n    for db_name, path in ((\"foo\", \"/foo.json\"), (\"foo-bar\", \"/foo-bar.json\")):\n        data = app_client_conflicting_database_names.get(path).json\n        assert db_name == data[\"database\"]\n\n\ndef test_inspect_file_used_for_count(app_client_immutable_and_inspect_file):\n    response = app_client_immutable_and_inspect_file.get(\n        \"/fixtures/sortable.json?_extra=count\"\n    )\n    assert response.json[\"count\"] == 100\n\n\n@pytest.mark.asyncio\nasync def test_http_options_request(ds_client):\n    response = await ds_client.options(\"/fixtures\")\n    assert response.status_code == 200\n    assert response.text == \"ok\"\n\n\n@pytest.mark.asyncio\nasync def test_db_path(app_client):\n    # Needs app_client because needs file based database\n    db = app_client.ds.get_database()\n    path = pathlib.Path(db.path)\n\n    assert path.exists()\n\n    datasette = Datasette([path])\n\n    # Previously this broke if path was a pathlib.Path:\n    await datasette.refresh_schemas()\n\n\n@pytest.mark.asyncio\nasync def test_hidden_sqlite_stat1_table():\n    ds = Datasette()\n    db = ds.add_memory_database(\"db\")\n    await db.execute_write(\"create table normal (id integer primary key, name text)\")\n    await db.execute_write(\"create index idx on normal (name)\")\n    await db.execute_write(\"analyze\")\n    data = (await ds.client.get(\"/db.json?_show_hidden=1\")).json()\n    tables = [(t[\"name\"], t[\"hidden\"]) for t in data[\"tables\"]]\n    assert tables in (\n        [(\"normal\", False), (\"sqlite_stat1\", True)],\n        [(\"normal\", False), (\"sqlite_stat1\", True), (\"sqlite_stat4\", True)],\n    )\n\n\n@pytest.mark.asyncio\nasync def test_hide_tables_starting_with_underscore():\n    ds = Datasette()\n    db = ds.add_memory_database(\"test_hide_tables_starting_with_underscore\")\n    await db.execute_write(\"create table normal (id integer primary key, name text)\")\n    await db.execute_write(\"create table _hidden (id integer primary key, name text)\")\n    data = (\n        await ds.client.get(\n            \"/test_hide_tables_starting_with_underscore.json?_show_hidden=1\"\n        )\n    ).json()\n    tables = [(t[\"name\"], t[\"hidden\"]) for t in data[\"tables\"]]\n    assert tables == [(\"normal\", False), (\"_hidden\", True)]\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\"db_name\", (\"foo\", r\"fo%o\", \"f~/c.d\"))\nasync def test_tilde_encoded_database_names(db_name):\n    ds = Datasette()\n    ds.add_memory_database(db_name)\n    response = await ds.client.get(\"/.json\")\n    assert db_name in response.json()[\"databases\"].keys()\n    path = response.json()[\"databases\"][db_name][\"path\"]\n    # And the JSON for that database\n    response2 = await ds.client.get(path + \".json\")\n    assert response2.status_code == 200\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"config,expected\",\n    (\n        ({}, {}),\n        ({\"plugins\": {\"datasette-foo\": \"bar\"}}, {\"plugins\": {\"datasette-foo\": \"bar\"}}),\n        # Test redaction\n        (\n            {\n                \"plugins\": {\n                    \"datasette-auth\": {\"secret_key\": \"key\"},\n                    \"datasette-foo\": \"bar\",\n                    \"datasette-auth2\": {\"password\": \"password\"},\n                    \"datasette-sentry\": {\n                        \"dsn\": \"sentry:///foo\",\n                    },\n                }\n            },\n            {\n                \"plugins\": {\n                    \"datasette-auth\": {\"secret_key\": \"***\"},\n                    \"datasette-foo\": \"bar\",\n                    \"datasette-auth2\": {\"password\": \"***\"},\n                    \"datasette-sentry\": {\"dsn\": \"***\"},\n                }\n            },\n        ),\n    ),\n)\nasync def test_config_json(config, expected):\n    \"/-/config.json should return redacted configuration\"\n    ds = Datasette(config=config)\n    response = await ds.client.get(\"/-/config.json\")\n    assert response.json() == expected\n\n\n@pytest.mark.asyncio\n@pytest.mark.skip(reason=\"rm?\")\n@pytest.mark.parametrize(\n    \"metadata,expected_config,expected_metadata\",\n    (\n        ({}, {}, {}),\n        (\n            # Metadata input\n            {\n                \"title\": \"Datasette Fixtures\",\n                \"databases\": {\n                    \"fixtures\": {\n                        \"tables\": {\n                            \"sortable\": {\n                                \"sortable_columns\": [\n                                    \"sortable\",\n                                    \"sortable_with_nulls\",\n                                    \"sortable_with_nulls_2\",\n                                    \"text\",\n                                ],\n                            },\n                            \"no_primary_key\": {\"sortable_columns\": [], \"hidden\": True},\n                            \"primary_key_multiple_columns_explicit_label\": {\n                                \"label_column\": \"content2\"\n                            },\n                            \"simple_view\": {\"sortable_columns\": [\"content\"]},\n                            \"searchable_view_configured_by_metadata\": {\n                                \"fts_table\": \"searchable_fts\",\n                                \"fts_pk\": \"pk\",\n                            },\n                            \"roadside_attractions\": {\n                                \"columns\": {\n                                    \"name\": \"The name of the attraction\",\n                                    \"address\": \"The street address for the attraction\",\n                                }\n                            },\n                            \"attraction_characteristic\": {\"sort_desc\": \"pk\"},\n                            \"facet_cities\": {\"sort\": \"name\"},\n                            \"paginated_view\": {\"size\": 25},\n                        },\n                    }\n                },\n            },\n            # Should produce a config with just the table configuration keys\n            {\n                \"databases\": {\n                    \"fixtures\": {\n                        \"tables\": {\n                            \"sortable\": {\n                                \"sortable_columns\": [\n                                    \"sortable\",\n                                    \"sortable_with_nulls\",\n                                    \"sortable_with_nulls_2\",\n                                    \"text\",\n                                ]\n                            },\n                            # These one get redacted:\n                            \"no_primary_key\": \"***\",\n                            \"primary_key_multiple_columns_explicit_label\": \"***\",\n                            \"simple_view\": {\"sortable_columns\": [\"content\"]},\n                            \"searchable_view_configured_by_metadata\": {\n                                \"fts_table\": \"searchable_fts\",\n                                \"fts_pk\": \"pk\",\n                            },\n                            \"attraction_characteristic\": {\"sort_desc\": \"pk\"},\n                            \"facet_cities\": {\"sort\": \"name\"},\n                            \"paginated_view\": {\"size\": 25},\n                        }\n                    }\n                }\n            },\n            # And metadata with everything else\n            {\n                \"title\": \"Datasette Fixtures\",\n                \"databases\": {\n                    \"fixtures\": {\n                        \"tables\": {\n                            \"roadside_attractions\": {\n                                \"columns\": {\n                                    \"name\": \"The name of the attraction\",\n                                    \"address\": \"The street address for the attraction\",\n                                }\n                            },\n                        }\n                    }\n                },\n            },\n        ),\n    ),\n)\nasync def test_upgrade_metadata(metadata, expected_config, expected_metadata):\n    ds = Datasette(metadata=metadata)\n    response = await ds.client.get(\"/-/config.json\")\n    assert response.json() == expected_config\n    response2 = await ds.client.get(\"/-/metadata.json\")\n    assert response2.json() == expected_metadata\n\n\nclass Either:\n    def __init__(self, a, b):\n        self.a = a\n        self.b = b\n\n    def __eq__(self, other):\n        return other == self.a or other == self.b\n"
  },
  {
    "path": "tests/test_api_write.py",
    "content": "from datasette.app import Datasette\nfrom datasette.utils import sqlite3\nfrom .utils import last_event\nimport pytest\nimport time\n\n\n@pytest.fixture\ndef ds_write(tmp_path_factory):\n    db_directory = tmp_path_factory.mktemp(\"dbs\")\n    db_path = str(db_directory / \"data.db\")\n    db_path_immutable = str(db_directory / \"immutable.db\")\n    db1 = sqlite3.connect(str(db_path))\n    db2 = sqlite3.connect(str(db_path_immutable))\n    for db in (db1, db2):\n        db.execute(\"vacuum\")\n        db.execute(\n            \"create table docs (id integer primary key, title text, score float, age integer)\"\n        )\n    ds = Datasette([db_path], immutables=[db_path_immutable])\n    ds.root_enabled = True\n    yield ds\n    # Close both setup connections plus any Datasette-managed connections.\n    db1.close()\n    db2.close()\n    for database in ds.databases.values():\n        if not database.is_memory:\n            database.close()\n\n\ndef write_token(ds, actor_id=\"root\", permissions=None):\n    to_sign = {\"a\": actor_id, \"token\": \"dstok\", \"t\": int(time.time())}\n    if permissions:\n        to_sign[\"_r\"] = {\"a\": permissions}\n    return \"dstok_{}\".format(ds.sign(to_sign, namespace=\"token\"))\n\n\ndef _headers(token):\n    return {\n        \"Authorization\": \"Bearer {}\".format(token),\n        \"Content-Type\": \"application/json\",\n    }\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"content_type\",\n    (\n        \"application/json\",\n        \"application/json; charset=utf-8\",\n    ),\n)\nasync def test_insert_row(ds_write, content_type):\n    token = write_token(ds_write)\n    response = await ds_write.client.post(\n        \"/data/docs/-/insert\",\n        json={\"row\": {\"title\": \"Test\", \"score\": 1.2, \"age\": 5}},\n        headers={\n            \"Authorization\": \"Bearer {}\".format(token),\n            \"Content-Type\": content_type,\n        },\n    )\n    expected_row = {\"id\": 1, \"title\": \"Test\", \"score\": 1.2, \"age\": 5}\n    assert response.status_code == 201\n    assert response.json()[\"ok\"] is True\n    assert response.json()[\"rows\"] == [expected_row]\n    rows = (await ds_write.get_database(\"data\").execute(\"select * from docs\")).dicts()\n    assert rows[0] == expected_row\n    # Analytics event\n    event = last_event(ds_write)\n    assert event.name == \"insert-rows\"\n    assert event.num_rows == 1\n    assert event.database == \"data\"\n    assert event.table == \"docs\"\n    assert not event.ignore\n    assert not event.replace\n\n\n@pytest.mark.asyncio\nasync def test_insert_row_alter(ds_write):\n    token = write_token(ds_write)\n    response = await ds_write.client.post(\n        \"/data/docs/-/insert\",\n        json={\n            \"row\": {\"title\": \"Test\", \"score\": 1.2, \"age\": 5, \"extra\": \"extra\"},\n            \"alter\": True,\n        },\n        headers=_headers(token),\n    )\n    assert response.status_code == 201\n    assert response.json()[\"ok\"] is True\n    assert response.json()[\"rows\"][0][\"extra\"] == \"extra\"\n    # Analytics event\n    event = last_event(ds_write)\n    assert event.name == \"alter-table\"\n    assert \"extra\" not in event.before_schema\n    assert \"extra\" in event.after_schema\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\"return_rows\", (True, False))\nasync def test_insert_rows(ds_write, return_rows):\n    token = write_token(ds_write)\n    data = {\n        \"rows\": [\n            {\"title\": \"Test {}\".format(i), \"score\": 1.0, \"age\": 5} for i in range(20)\n        ]\n    }\n    if return_rows:\n        data[\"return\"] = True\n    response = await ds_write.client.post(\n        \"/data/docs/-/insert\",\n        json=data,\n        headers=_headers(token),\n    )\n    assert response.status_code == 201\n\n    # Analytics event\n    event = last_event(ds_write)\n    assert event.name == \"insert-rows\"\n    assert event.num_rows == 20\n    assert event.database == \"data\"\n    assert event.table == \"docs\"\n    assert not event.ignore\n    assert not event.replace\n\n    actual_rows = (\n        await ds_write.get_database(\"data\").execute(\"select * from docs\")\n    ).dicts()\n    assert len(actual_rows) == 20\n    assert actual_rows == [\n        {\"id\": i + 1, \"title\": \"Test {}\".format(i), \"score\": 1.0, \"age\": 5}\n        for i in range(20)\n    ]\n    assert response.json()[\"ok\"] is True\n    if return_rows:\n        assert response.json()[\"rows\"] == actual_rows\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"path,input,special_case,expected_status,expected_errors\",\n    (\n        (\n            \"/data2/docs/-/insert\",\n            {},\n            None,\n            404,\n            [\"Database not found\"],\n        ),\n        (\n            \"/data/docs2/-/insert\",\n            {},\n            None,\n            404,\n            [\"Table not found\"],\n        ),\n        (\n            \"/data/docs/-/insert\",\n            {\"rows\": [{\"title\": \"Test\"} for i in range(10)]},\n            \"bad_token\",\n            403,\n            [\"Permission denied\"],\n        ),\n        (\n            \"/data/docs/-/insert\",\n            {},\n            \"invalid_json\",\n            400,\n            [\n                \"Invalid JSON: Expecting property name enclosed in double quotes: line 1 column 2 (char 1)\"\n            ],\n        ),\n        (\n            \"/data/docs/-/insert\",\n            {},\n            \"invalid_content_type\",\n            400,\n            [\"Invalid content-type, must be application/json\"],\n        ),\n        (\n            \"/data/docs/-/insert\",\n            [],\n            None,\n            400,\n            [\"JSON must be a dictionary\"],\n        ),\n        (\n            \"/data/docs/-/insert\",\n            {\"row\": \"blah\"},\n            None,\n            400,\n            ['\"row\" must be a dictionary'],\n        ),\n        (\n            \"/data/docs/-/insert\",\n            {\"blah\": \"blah\"},\n            None,\n            400,\n            ['JSON must have one or other of \"row\" or \"rows\"'],\n        ),\n        (\n            \"/data/docs/-/insert\",\n            {\"rows\": \"blah\"},\n            None,\n            400,\n            ['\"rows\" must be a list'],\n        ),\n        (\n            \"/data/docs/-/insert\",\n            {\"rows\": [\"blah\"]},\n            None,\n            400,\n            ['\"rows\" must be a list of dictionaries'],\n        ),\n        (\n            \"/data/docs/-/insert\",\n            {\"rows\": [{\"title\": \"Test\"} for i in range(101)]},\n            None,\n            400,\n            [\"Too many rows, maximum allowed is 100\"],\n        ),\n        (\n            \"/data/docs/-/insert\",\n            {\"rows\": [{\"id\": 1, \"title\": \"Test\"}, {\"id\": 2, \"title\": \"Test\"}]},\n            \"duplicate_id\",\n            400,\n            [\"UNIQUE constraint failed: docs.id\"],\n        ),\n        (\n            \"/data/docs/-/insert\",\n            {\"rows\": [{\"title\": \"Test\"}], \"ignore\": True, \"replace\": True},\n            None,\n            400,\n            ['Cannot use \"ignore\" and \"replace\" at the same time'],\n        ),\n        (\n            # Replace is not allowed if you don't have update-row\n            \"/data/docs/-/insert\",\n            {\"rows\": [{\"title\": \"Test\"}], \"replace\": True},\n            \"insert-but-not-update\",\n            403,\n            ['Permission denied: need update-row to use \"replace\"'],\n        ),\n        (\n            \"/data/docs/-/insert\",\n            {\"rows\": [{\"title\": \"Test\"}], \"invalid_param\": True},\n            None,\n            400,\n            ['Invalid parameter: \"invalid_param\"'],\n        ),\n        (\n            \"/data/docs/-/insert\",\n            {\"rows\": [{\"title\": \"Test\"}], \"one\": True, \"two\": True},\n            None,\n            400,\n            ['Invalid parameter: \"one\", \"two\"'],\n        ),\n        (\n            \"/immutable/docs/-/insert\",\n            {\"rows\": [{\"title\": \"Test\"}]},\n            None,\n            403,\n            [\"Database is immutable\"],\n        ),\n        # Validate columns of each row\n        (\n            \"/data/docs/-/insert\",\n            {\"rows\": [{\"title\": \"Test\", \"bad\": 1, \"worse\": 2} for i in range(2)]},\n            None,\n            400,\n            [\n                \"Row 0 has invalid columns: bad, worse\",\n                \"Row 1 has invalid columns: bad, worse\",\n            ],\n        ),\n        ## UPSERT ERRORS:\n        (\n            \"/immutable/docs/-/upsert\",\n            {\"rows\": [{\"title\": \"Test\"}]},\n            None,\n            403,\n            [\"Database is immutable\"],\n        ),\n        (\n            \"/data/badtable/-/upsert\",\n            {\"rows\": [{\"title\": \"Test\"}]},\n            None,\n            404,\n            [\"Table not found\"],\n        ),\n        # missing primary key\n        (\n            \"/data/docs/-/upsert\",\n            {\"rows\": [{\"title\": \"Missing PK\"}]},\n            None,\n            400,\n            ['Row 0 is missing primary key column(s): \"id\"'],\n        ),\n        # Upsert does not support ignore or replace\n        (\n            \"/data/docs/-/upsert\",\n            {\"rows\": [{\"id\": 1, \"title\": \"Bad\"}], \"ignore\": True},\n            None,\n            400,\n            [\"Upsert does not support ignore or replace\"],\n        ),\n        # Upsert permissions\n        (\n            \"/data/docs/-/upsert\",\n            {\"rows\": [{\"id\": 1, \"title\": \"Disallowed\"}]},\n            \"insert-but-not-update\",\n            403,\n            [\"Permission denied: need both insert-row and update-row\"],\n        ),\n        (\n            \"/data/docs/-/upsert\",\n            {\"rows\": [{\"id\": 1, \"title\": \"Disallowed\"}]},\n            \"update-but-not-insert\",\n            403,\n            [\"Permission denied: need both insert-row and update-row\"],\n        ),\n        # Alter table forbidden without alter permission\n        (\n            \"/data/docs/-/upsert\",\n            {\"rows\": [{\"id\": 1, \"title\": \"One\", \"extra\": \"extra\"}], \"alter\": True},\n            \"update-and-insert-but-no-alter\",\n            403,\n            [\"Permission denied for alter-table\"],\n        ),\n    ),\n)\nasync def test_insert_or_upsert_row_errors(\n    ds_write, path, input, special_case, expected_status, expected_errors\n):\n    token_permissions = []\n    if special_case == \"insert-but-not-update\":\n        token_permissions = [\"ir\", \"vi\"]\n    if special_case == \"update-but-not-insert\":\n        token_permissions = [\"ur\", \"vi\"]\n    if special_case == \"update-and-insert-but-no-alter\":\n        token_permissions = [\"ur\", \"ir\"]\n    token = write_token(ds_write, permissions=token_permissions)\n    if special_case == \"duplicate_id\":\n        await ds_write.get_database(\"data\").execute_write(\n            \"insert into docs (id) values (1)\"\n        )\n    if special_case == \"bad_token\":\n        token += \"bad\"\n    kwargs = dict(\n        json=input,\n        headers={\n            \"Authorization\": \"Bearer {}\".format(token),\n            \"Content-Type\": (\n                \"text/plain\"\n                if special_case == \"invalid_content_type\"\n                else \"application/json\"\n            ),\n        },\n    )\n\n    actor_response = (\n        await ds_write.client.get(\"/-/actor.json\", headers=kwargs[\"headers\"])\n    ).json()\n    assert set((actor_response[\"actor\"] or {}).get(\"_r\", {}).get(\"a\") or []) == set(\n        token_permissions\n    )\n\n    if special_case == \"invalid_json\":\n        del kwargs[\"json\"]\n        kwargs[\"content\"] = \"{bad json\"\n    before_count = (\n        await ds_write.get_database(\"data\").execute(\"select count(*) from docs\")\n    ).rows[0][0] == 0\n    response = await ds_write.client.post(\n        path,\n        **kwargs,\n    )\n    assert response.status_code == expected_status\n    assert response.json()[\"ok\"] is False\n    assert response.json()[\"errors\"] == expected_errors\n    # Check that no rows were inserted\n    after_count = (\n        await ds_write.get_database(\"data\").execute(\"select count(*) from docs\")\n    ).rows[0][0] == 0\n    assert before_count == after_count\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\"allowed\", (True, False))\nasync def test_upsert_permissions_per_table(ds_write, allowed):\n    # https://github.com/simonw/datasette/issues/2262\n    token = \"dstok_{}\".format(\n        ds_write.sign(\n            {\n                \"a\": \"root\",\n                \"token\": \"dstok\",\n                \"t\": int(time.time()),\n                \"_r\": {\n                    \"r\": {\n                        \"data\": {\n                            \"docs\" if allowed else \"other\": [\"ir\", \"ur\"],\n                        }\n                    }\n                },\n            },\n            namespace=\"token\",\n        )\n    )\n    response = await ds_write.client.post(\n        \"/data/docs/-/upsert\",\n        json={\"rows\": [{\"id\": 1, \"title\": \"One\"}]},\n        headers={\n            \"Authorization\": \"Bearer {}\".format(token),\n        },\n    )\n    if allowed:\n        assert response.status_code == 200\n        assert response.json()[\"ok\"] is True\n    else:\n        assert response.status_code == 403\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"ignore,replace,expected_rows\",\n    (\n        (\n            True,\n            False,\n            [\n                {\"id\": 1, \"title\": \"Exists\", \"score\": None, \"age\": None},\n            ],\n        ),\n        (\n            False,\n            True,\n            [\n                {\"id\": 1, \"title\": \"One\", \"score\": None, \"age\": None},\n            ],\n        ),\n    ),\n)\n@pytest.mark.parametrize(\"should_return\", (True, False))\nasync def test_insert_ignore_replace(\n    ds_write, ignore, replace, expected_rows, should_return\n):\n    await ds_write.get_database(\"data\").execute_write(\n        \"insert into docs (id, title) values (1, 'Exists')\"\n    )\n    token = write_token(ds_write)\n    data = {\"rows\": [{\"id\": 1, \"title\": \"One\"}]}\n    if ignore:\n        data[\"ignore\"] = True\n    if replace:\n        data[\"replace\"] = True\n    if should_return:\n        data[\"return\"] = True\n    response = await ds_write.client.post(\n        \"/data/docs/-/insert\",\n        json=data,\n        headers=_headers(token),\n    )\n    assert response.status_code == 201\n\n    # Analytics event\n    event = last_event(ds_write)\n    assert event.name == \"insert-rows\"\n    assert event.num_rows == 1\n    assert event.database == \"data\"\n    assert event.table == \"docs\"\n    assert event.ignore == ignore\n    assert event.replace == replace\n\n    actual_rows = (\n        await ds_write.get_database(\"data\").execute(\"select * from docs\")\n    ).dicts()\n\n    assert actual_rows == expected_rows\n    assert response.json()[\"ok\"] is True\n    if should_return:\n        assert response.json()[\"rows\"] == expected_rows\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"initial,input,expected_rows\",\n    (\n        (\n            # Simple primary key update\n            {\"rows\": [{\"id\": 1, \"title\": \"One\"}], \"pk\": \"id\"},\n            {\"rows\": [{\"id\": 1, \"title\": \"Two\"}]},\n            [\n                {\"id\": 1, \"title\": \"Two\"},\n            ],\n        ),\n        (\n            # Multiple rows update one of them\n            {\n                \"rows\": [{\"id\": 1, \"title\": \"One\"}, {\"id\": 2, \"title\": \"Two\"}],\n                \"pk\": \"id\",\n            },\n            {\"rows\": [{\"id\": 1, \"title\": \"Three\"}]},\n            [\n                {\"id\": 1, \"title\": \"Three\"},\n                {\"id\": 2, \"title\": \"Two\"},\n            ],\n        ),\n        (\n            # rowid update\n            {\"rows\": [{\"title\": \"One\"}]},\n            {\"rows\": [{\"rowid\": 1, \"title\": \"Two\"}]},\n            [\n                {\"rowid\": 1, \"title\": \"Two\"},\n            ],\n        ),\n        (\n            # Compound primary key update\n            {\"rows\": [{\"id\": 1, \"title\": \"One\", \"score\": 1}], \"pks\": [\"id\", \"score\"]},\n            {\"rows\": [{\"id\": 1, \"title\": \"Two\", \"score\": 1}]},\n            [\n                {\"id\": 1, \"title\": \"Two\", \"score\": 1},\n            ],\n        ),\n        (\n            # Upsert with an alter\n            {\"rows\": [{\"id\": 1, \"title\": \"One\"}], \"pk\": \"id\"},\n            {\"rows\": [{\"id\": 1, \"title\": \"Two\", \"extra\": \"extra\"}], \"alter\": True},\n            [{\"id\": 1, \"title\": \"Two\", \"extra\": \"extra\"}],\n        ),\n    ),\n)\n@pytest.mark.parametrize(\"should_return\", (False, True))\nasync def test_upsert(ds_write, initial, input, expected_rows, should_return):\n    token = write_token(ds_write)\n    # Insert initial data\n    initial[\"table\"] = \"upsert_test\"\n    create_response = await ds_write.client.post(\n        \"/data/-/create\",\n        json=initial,\n        headers=_headers(token),\n    )\n    assert create_response.status_code == 201\n    if should_return:\n        input[\"return\"] = True\n    response = await ds_write.client.post(\n        \"/data/upsert_test/-/upsert\",\n        json=input,\n        headers=_headers(token),\n    )\n    assert response.status_code == 200, response.text\n    assert response.json()[\"ok\"] is True\n\n    # Analytics event\n    event = last_event(ds_write)\n    assert event.database == \"data\"\n    assert event.table == \"upsert_test\"\n    if input.get(\"alter\"):\n        assert event.name == \"alter-table\"\n        assert \"extra\" in event.after_schema\n    else:\n        assert event.name == \"upsert-rows\"\n        assert event.num_rows == 1\n\n    if should_return:\n        # We only expect it to return rows corresponding to those we sent\n        expected_returned_rows = expected_rows[: len(input[\"rows\"])]\n        assert response.json()[\"rows\"] == expected_returned_rows\n    # Check the database too\n    actual_rows = (\n        await ds_write.client.get(\"/data/upsert_test.json?_shape=array\")\n    ).json()\n    assert actual_rows == expected_rows\n    # Drop the upsert_test table\n    await ds_write.get_database(\"data\").execute_write(\"drop table upsert_test\")\n\n\nasync def _insert_row(ds):\n    insert_response = await ds.client.post(\n        \"/data/docs/-/insert\",\n        json={\"row\": {\"title\": \"Row one\", \"score\": 1.2, \"age\": 5}, \"return\": True},\n        headers=_headers(write_token(ds)),\n    )\n    assert insert_response.status_code == 201\n    return insert_response.json()[\"rows\"][0][\"id\"]\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\"scenario\", (\"no_token\", \"no_perm\", \"bad_table\"))\nasync def test_delete_row_errors(ds_write, scenario):\n    if scenario == \"no_token\":\n        token = \"bad_token\"\n    elif scenario == \"no_perm\":\n        token = write_token(ds_write, actor_id=\"not-root\")\n    else:\n        token = write_token(ds_write)\n\n    pk = await _insert_row(ds_write)\n\n    path = \"/data/{}/{}/-/delete\".format(\n        \"docs\" if scenario != \"bad_table\" else \"bad_table\", pk\n    )\n    response = await ds_write.client.post(\n        path,\n        headers=_headers(token),\n    )\n    assert response.status_code == 403 if scenario in (\"no_token\", \"bad_token\") else 404\n    assert response.json()[\"ok\"] is False\n    assert (\n        response.json()[\"errors\"] == [\"Permission denied\"]\n        if scenario == \"no_token\"\n        else [\"Table not found\"]\n    )\n    assert len((await ds_write.client.get(\"/data/docs.json?_shape=array\")).json()) == 1\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"table,row_for_create,pks,delete_path\",\n    (\n        (\"rowid_table\", {\"name\": \"rowid row\"}, None, None),\n        (\"pk_table\", {\"id\": 1, \"name\": \"ID table\"}, \"id\", \"1\"),\n        (\n            \"compound_pk_table\",\n            {\"type\": \"article\", \"key\": \"k\"},\n            [\"type\", \"key\"],\n            \"article,k\",\n        ),\n    ),\n)\nasync def test_delete_row(ds_write, table, row_for_create, pks, delete_path):\n    # First create the table with that example row\n    create_data = {\n        \"table\": table,\n        \"row\": row_for_create,\n    }\n    if pks:\n        if isinstance(pks, str):\n            create_data[\"pk\"] = pks\n        else:\n            create_data[\"pks\"] = pks\n    create_response = await ds_write.client.post(\n        \"/data/-/create\",\n        json=create_data,\n        headers=_headers(write_token(ds_write)),\n    )\n    assert create_response.status_code == 201, create_response.json()\n    # Should be a single row\n    assert (\n        await ds_write.client.get(\n            \"/data/-/query.json?_shape=arrayfirst&sql=select+count(*)+from+{}\".format(\n                table\n            )\n        )\n    ).json() == [1]\n    # Now delete the row\n    if delete_path is None:\n        # Special case for that rowid table\n        delete_path = (\n            await ds_write.client.get(\n                \"/data/-/query.json?_shape=arrayfirst&sql=select+rowid+from+{}\".format(\n                    table\n                )\n            )\n        ).json()[0]\n\n    delete_response = await ds_write.client.post(\n        \"/data/{}/{}/-/delete\".format(table, delete_path),\n        headers=_headers(write_token(ds_write)),\n    )\n    assert delete_response.status_code == 200\n\n    # Analytics event\n    event = last_event(ds_write)\n    assert event.name == \"delete-row\"\n    assert event.database == \"data\"\n    assert event.table == table\n    assert event.pks == str(delete_path).split(\",\")\n    assert (\n        await ds_write.client.get(\n            \"/data/-/query.json?_shape=arrayfirst&sql=select+count(*)+from+{}\".format(\n                table\n            )\n        )\n    ).json() == [0]\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"scenario\", (\"no_token\", \"no_perm\", \"bad_table\", \"cannot_alter\")\n)\nasync def test_update_row_check_permission(ds_write, scenario):\n    if scenario == \"no_token\":\n        token = \"bad_token\"\n    elif scenario == \"no_perm\":\n        token = write_token(ds_write, actor_id=\"not-root\")\n    elif scenario == \"cannot_alter\":\n        # update-row but no alter-table:\n        token = write_token(ds_write, permissions=[\"ur\"])\n    else:\n        token = write_token(ds_write)\n\n    pk = await _insert_row(ds_write)\n\n    path = \"/data/{}/{}/-/update\".format(\n        \"docs\" if scenario != \"bad_table\" else \"bad_table\", pk\n    )\n\n    json_body = {\"update\": {\"title\": \"New title\"}}\n    if scenario == \"cannot_alter\":\n        json_body[\"alter\"] = True\n\n    response = await ds_write.client.post(\n        path,\n        json=json_body,\n        headers=_headers(token),\n    )\n    assert response.status_code == 403 if scenario in (\"no_token\", \"bad_token\") else 404\n    assert response.json()[\"ok\"] is False\n    assert (\n        response.json()[\"errors\"] == [\"Permission denied\"]\n        if scenario == \"no_token\"\n        else [\"Table not found\"]\n    )\n\n\n@pytest.mark.asyncio\nasync def test_update_row_invalid_key(ds_write):\n    token = write_token(ds_write)\n\n    pk = await _insert_row(ds_write)\n\n    path = \"/data/docs/{}/-/update\".format(pk)\n    response = await ds_write.client.post(\n        path,\n        json={\"update\": {\"title\": \"New title\"}, \"bad_key\": 1},\n        headers=_headers(token),\n    )\n    assert response.status_code == 400\n    assert response.json() == {\"ok\": False, \"errors\": [\"Invalid keys: bad_key\"]}\n\n\n@pytest.mark.asyncio\nasync def test_update_row_alter(ds_write):\n    token = write_token(ds_write, permissions=[\"ur\", \"at\"])\n    pk = await _insert_row(ds_write)\n    path = \"/data/docs/{}/-/update\".format(pk)\n    response = await ds_write.client.post(\n        path,\n        json={\"update\": {\"title\": \"New title\", \"extra\": \"extra\"}, \"alter\": True},\n        headers=_headers(token),\n    )\n    assert response.status_code == 200\n    assert response.json() == {\"ok\": True}\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"input,expected_errors\",\n    (\n        ({\"title\": \"New title\"}, None),\n        ({\"title\": None}, None),\n        ({\"score\": 1.6}, None),\n        ({\"age\": 10}, None),\n        ({\"title\": \"New title\", \"score\": 1.6}, None),\n        ({\"title2\": \"New title\"}, [\"no such column: title2\"]),\n    ),\n)\n@pytest.mark.parametrize(\"use_return\", (True, False))\nasync def test_update_row(ds_write, input, expected_errors, use_return):\n    token = write_token(ds_write)\n    pk = await _insert_row(ds_write)\n\n    path = \"/data/docs/{}/-/update\".format(pk)\n\n    data = {\"update\": input}\n    if use_return:\n        data[\"return\"] = True\n\n    response = await ds_write.client.post(\n        path,\n        json=data,\n        headers=_headers(token),\n    )\n    if expected_errors:\n        assert response.status_code == 400\n        assert response.json()[\"ok\"] is False\n        assert response.json()[\"errors\"] == expected_errors\n        return\n\n    assert response.json()[\"ok\"] is True\n    if not use_return:\n        assert \"row\" not in response.json()\n    else:\n        returned_row = response.json()[\"row\"]\n        assert returned_row[\"id\"] == pk\n        for k, v in input.items():\n            assert returned_row[k] == v\n\n    # Analytics event\n    event = last_event(ds_write)\n    assert event.actor == {\"id\": \"root\", \"token\": \"dstok\"}\n    assert event.database == \"data\"\n    assert event.table == \"docs\"\n    assert event.pks == [str(pk)]\n\n    # And fetch the row to check it's updated\n    response = await ds_write.client.get(\n        \"/data/docs/{}.json?_shape=array\".format(pk),\n    )\n    assert response.status_code == 200\n    row = response.json()[0]\n    assert row[\"id\"] == pk\n    for k, v in input.items():\n        assert row[k] == v\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"scenario\", (\"no_token\", \"no_perm\", \"bad_table\", \"has_perm\", \"immutable\")\n)\nasync def test_drop_table(ds_write, scenario):\n    if scenario == \"no_token\":\n        token = \"bad_token\"\n    elif scenario == \"no_perm\":\n        token = write_token(ds_write, actor_id=\"not-root\")\n    else:\n        token = write_token(ds_write)\n    should_work = scenario == \"has_perm\"\n    await ds_write.get_database(\"data\").execute_write(\n        \"insert into docs (id, title) values (1, 'Row 1')\"\n    )\n    path = \"/{database}/{table}/-/drop\".format(\n        database=\"immutable\" if scenario == \"immutable\" else \"data\",\n        table=\"docs\" if scenario != \"bad_table\" else \"bad_table\",\n    )\n    response = await ds_write.client.post(\n        path,\n        headers=_headers(token),\n    )\n    if not should_work:\n        assert (\n            response.status_code == 403\n            if scenario in (\"no_token\", \"bad_token\")\n            else 404\n        )\n        assert response.json()[\"ok\"] is False\n        expected_error = \"Permission denied\"\n        if scenario == \"bad_table\":\n            expected_error = \"Table not found\"\n        elif scenario == \"immutable\":\n            expected_error = \"Database is immutable\"\n        assert response.json()[\"errors\"] == [expected_error]\n        assert (await ds_write.client.get(\"/data/docs\")).status_code == 200\n    else:\n        # It should show a confirmation page\n        assert response.status_code == 200\n        assert response.json() == {\n            \"ok\": True,\n            \"database\": \"data\",\n            \"table\": \"docs\",\n            \"row_count\": 1,\n            \"message\": 'Pass \"confirm\": true to confirm',\n        }\n        assert (await ds_write.client.get(\"/data/docs\")).status_code == 200\n        # Now send confirm: true\n        response2 = await ds_write.client.post(\n            path,\n            json={\"confirm\": True},\n            headers=_headers(token),\n        )\n        assert response2.json() == {\"ok\": True}\n        # Check event\n        event = last_event(ds_write)\n        assert event.name == \"drop-table\"\n        assert event.actor == {\"id\": \"root\", \"token\": \"dstok\"}\n        assert event.table == \"docs\"\n        assert event.database == \"data\"\n        # Table should 404\n        assert (await ds_write.client.get(\"/data/docs\")).status_code == 404\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"input,expected_status,expected_response,expected_events\",\n    (\n        # Permission error with a bad token\n        (\n            {\"table\": \"bad\", \"row\": {\"id\": 1}},\n            403,\n            {\"ok\": False, \"errors\": [\"Permission denied\"]},\n            [],\n        ),\n        # Successful creation with columns:\n        (\n            {\n                \"table\": \"one\",\n                \"columns\": [\n                    {\n                        \"name\": \"id\",\n                        \"type\": \"integer\",\n                    },\n                    {\n                        \"name\": \"title\",\n                        \"type\": \"text\",\n                    },\n                    {\n                        \"name\": \"score\",\n                        \"type\": \"integer\",\n                    },\n                    {\n                        \"name\": \"weight\",\n                        \"type\": \"float\",\n                    },\n                    {\n                        \"name\": \"thumbnail\",\n                        \"type\": \"blob\",\n                    },\n                ],\n                \"pk\": \"id\",\n            },\n            201,\n            {\n                \"ok\": True,\n                \"database\": \"data\",\n                \"table\": \"one\",\n                \"table_url\": \"http://localhost/data/one\",\n                \"table_api_url\": \"http://localhost/data/one.json\",\n                \"schema\": (\n                    \"CREATE TABLE [one] (\\n\"\n                    \"   [id] INTEGER PRIMARY KEY,\\n\"\n                    \"   [title] TEXT,\\n\"\n                    \"   [score] INTEGER,\\n\"\n                    \"   [weight] FLOAT,\\n\"\n                    \"   [thumbnail] BLOB\\n\"\n                    \")\"\n                ),\n            },\n            [\"create-table\"],\n        ),\n        # Successful creation with rows:\n        (\n            {\n                \"table\": \"two\",\n                \"rows\": [\n                    {\n                        \"id\": 1,\n                        \"title\": \"Row 1\",\n                        \"score\": 1.5,\n                    },\n                    {\n                        \"id\": 2,\n                        \"title\": \"Row 2\",\n                        \"score\": 1.5,\n                    },\n                ],\n                \"pk\": \"id\",\n            },\n            201,\n            {\n                \"ok\": True,\n                \"database\": \"data\",\n                \"table\": \"two\",\n                \"table_url\": \"http://localhost/data/two\",\n                \"table_api_url\": \"http://localhost/data/two.json\",\n                \"schema\": (\n                    \"CREATE TABLE [two] (\\n\"\n                    \"   [id] INTEGER PRIMARY KEY,\\n\"\n                    \"   [title] TEXT,\\n\"\n                    \"   [score] FLOAT\\n\"\n                    \")\"\n                ),\n                \"row_count\": 2,\n            },\n            [\"create-table\", \"insert-rows\"],\n        ),\n        # Successful creation with row:\n        (\n            {\n                \"table\": \"three\",\n                \"row\": {\n                    \"id\": 1,\n                    \"title\": \"Row 1\",\n                    \"score\": 1.5,\n                },\n                \"pk\": \"id\",\n            },\n            201,\n            {\n                \"ok\": True,\n                \"database\": \"data\",\n                \"table\": \"three\",\n                \"table_url\": \"http://localhost/data/three\",\n                \"table_api_url\": \"http://localhost/data/three.json\",\n                \"schema\": (\n                    \"CREATE TABLE [three] (\\n\"\n                    \"   [id] INTEGER PRIMARY KEY,\\n\"\n                    \"   [title] TEXT,\\n\"\n                    \"   [score] FLOAT\\n\"\n                    \")\"\n                ),\n                \"row_count\": 1,\n            },\n            [\"create-table\", \"insert-rows\"],\n        ),\n        # Create with row and no primary key\n        (\n            {\n                \"table\": \"four\",\n                \"row\": {\n                    \"name\": \"Row 1\",\n                },\n            },\n            201,\n            {\n                \"ok\": True,\n                \"database\": \"data\",\n                \"table\": \"four\",\n                \"table_url\": \"http://localhost/data/four\",\n                \"table_api_url\": \"http://localhost/data/four.json\",\n                \"schema\": (\"CREATE TABLE [four] (\\n\" \"   [name] TEXT\\n\" \")\"),\n                \"row_count\": 1,\n            },\n            [\"create-table\", \"insert-rows\"],\n        ),\n        # Create table with compound primary key\n        (\n            {\n                \"table\": \"five\",\n                \"row\": {\"type\": \"article\", \"key\": 123, \"title\": \"Article 1\"},\n                \"pks\": [\"type\", \"key\"],\n            },\n            201,\n            {\n                \"ok\": True,\n                \"database\": \"data\",\n                \"table\": \"five\",\n                \"table_url\": \"http://localhost/data/five\",\n                \"table_api_url\": \"http://localhost/data/five.json\",\n                \"schema\": (\n                    \"CREATE TABLE [five] (\\n   [type] TEXT,\\n   [key] INTEGER,\\n\"\n                    \"   [title] TEXT,\\n   PRIMARY KEY ([type], [key])\\n)\"\n                ),\n                \"row_count\": 1,\n            },\n            [\"create-table\", \"insert-rows\"],\n        ),\n        # Error: Table is required\n        (\n            {\n                \"row\": {\"id\": 1},\n            },\n            400,\n            {\n                \"ok\": False,\n                \"errors\": [\"Table is required\"],\n            },\n            [],\n        ),\n        # Error: Invalid table name\n        (\n            {\n                \"table\": \"sqlite_bad_name\",\n                \"row\": {\"id\": 1},\n            },\n            400,\n            {\n                \"ok\": False,\n                \"errors\": [\"Invalid table name\"],\n            },\n            [],\n        ),\n        # Error: JSON must be an object\n        (\n            [],\n            400,\n            {\n                \"ok\": False,\n                \"errors\": [\"JSON must be an object\"],\n            },\n            [],\n        ),\n        # Error: Cannot specify columns with rows or row\n        (\n            {\n                \"table\": \"bad\",\n                \"columns\": [{\"name\": \"id\", \"type\": \"integer\"}],\n                \"rows\": [{\"id\": 1}],\n            },\n            400,\n            {\n                \"ok\": False,\n                \"errors\": [\"Cannot specify columns with rows or row\"],\n            },\n            [],\n        ),\n        # Error: columns, rows or row is required\n        (\n            {\n                \"table\": \"bad\",\n            },\n            400,\n            {\n                \"ok\": False,\n                \"errors\": [\"columns, rows or row is required\"],\n            },\n            [],\n        ),\n        # Error: columns must be a list\n        (\n            {\n                \"table\": \"bad\",\n                \"columns\": {\"name\": \"id\", \"type\": \"integer\"},\n            },\n            400,\n            {\n                \"ok\": False,\n                \"errors\": [\"columns must be a list\"],\n            },\n            [],\n        ),\n        # Error: columns must be a list of objects\n        (\n            {\n                \"table\": \"bad\",\n                \"columns\": [\"id\"],\n            },\n            400,\n            {\n                \"ok\": False,\n                \"errors\": [\"columns must be a list of objects\"],\n            },\n            [],\n        ),\n        # Error: Column name is required\n        (\n            {\n                \"table\": \"bad\",\n                \"columns\": [{\"type\": \"integer\"}],\n            },\n            400,\n            {\n                \"ok\": False,\n                \"errors\": [\"Column name is required\"],\n            },\n            [],\n        ),\n        # Error: Unsupported column type\n        (\n            {\n                \"table\": \"bad\",\n                \"columns\": [{\"name\": \"id\", \"type\": \"bad\"}],\n            },\n            400,\n            {\n                \"ok\": False,\n                \"errors\": [\"Unsupported column type: bad\"],\n            },\n            [],\n        ),\n        # Error: Duplicate column name\n        (\n            {\n                \"table\": \"bad\",\n                \"columns\": [\n                    {\"name\": \"id\", \"type\": \"integer\"},\n                    {\"name\": \"id\", \"type\": \"integer\"},\n                ],\n            },\n            400,\n            {\n                \"ok\": False,\n                \"errors\": [\"Duplicate column name: id\"],\n            },\n            [],\n        ),\n        # Error: rows must be a list\n        (\n            {\n                \"table\": \"bad\",\n                \"rows\": {\"id\": 1},\n            },\n            400,\n            {\n                \"ok\": False,\n                \"errors\": [\"rows must be a list\"],\n            },\n            [],\n        ),\n        # Error: rows must be a list of objects\n        (\n            {\n                \"table\": \"bad\",\n                \"rows\": [\"id\"],\n            },\n            400,\n            {\n                \"ok\": False,\n                \"errors\": [\"rows must be a list of objects\"],\n            },\n            [],\n        ),\n        # Error: pk must be a string\n        (\n            {\n                \"table\": \"bad\",\n                \"row\": {\"id\": 1},\n                \"pk\": 1,\n            },\n            400,\n            {\n                \"ok\": False,\n                \"errors\": [\"pk must be a string\"],\n            },\n            [],\n        ),\n        # Error: Cannot specify both pk and pks\n        (\n            {\n                \"table\": \"bad\",\n                \"row\": {\"id\": 1, \"name\": \"Row 1\"},\n                \"pk\": \"id\",\n                \"pks\": [\"id\", \"name\"],\n            },\n            400,\n            {\n                \"ok\": False,\n                \"errors\": [\"Cannot specify both pk and pks\"],\n            },\n            [],\n        ),\n        # Error: pks must be a list\n        (\n            {\n                \"table\": \"bad\",\n                \"row\": {\"id\": 1, \"name\": \"Row 1\"},\n                \"pks\": \"id\",\n            },\n            400,\n            {\n                \"ok\": False,\n                \"errors\": [\"pks must be a list\"],\n            },\n            [],\n        ),\n        # Error: pks must be a list of strings\n        (\n            {\"table\": \"bad\", \"row\": {\"id\": 1, \"name\": \"Row 1\"}, \"pks\": [1, 2]},\n            400,\n            {\"ok\": False, \"errors\": [\"pks must be a list of strings\"]},\n            [],\n        ),\n        # Error: ignore and replace are mutually exclusive\n        (\n            {\n                \"table\": \"bad\",\n                \"row\": {\"id\": 1, \"name\": \"Row 1\"},\n                \"pk\": \"id\",\n                \"ignore\": True,\n                \"replace\": True,\n            },\n            400,\n            {\n                \"ok\": False,\n                \"errors\": [\"ignore and replace are mutually exclusive\"],\n            },\n            [],\n        ),\n        # ignore and replace require row or rows\n        (\n            {\n                \"table\": \"bad\",\n                \"columns\": [{\"name\": \"id\", \"type\": \"integer\"}],\n                \"ignore\": True,\n            },\n            400,\n            {\n                \"ok\": False,\n                \"errors\": [\"ignore and replace require row or rows\"],\n            },\n            [],\n        ),\n        # ignore and replace require pk or pks\n        (\n            {\n                \"table\": \"bad\",\n                \"row\": {\"id\": 1},\n                \"ignore\": True,\n            },\n            400,\n            {\n                \"ok\": False,\n                \"errors\": [\"ignore and replace require pk or pks\"],\n            },\n            [],\n        ),\n        (\n            {\n                \"table\": \"bad\",\n                \"row\": {\"id\": 1},\n                \"replace\": True,\n            },\n            400,\n            {\n                \"ok\": False,\n                \"errors\": [\"ignore and replace require pk or pks\"],\n            },\n            [],\n        ),\n    ),\n)\nasync def test_create_table(\n    ds_write, input, expected_status, expected_response, expected_events\n):\n    ds_write._tracked_events = []\n    # Special case for expected status of 403\n    if expected_status == 403:\n        token = \"bad_token\"\n    else:\n        token = write_token(ds_write)\n    response = await ds_write.client.post(\n        \"/data/-/create\",\n        json=input,\n        headers=_headers(token),\n    )\n    assert response.status_code == expected_status\n    data = response.json()\n    assert data == expected_response\n    # Should have tracked the expected events\n    events = ds_write._tracked_events\n    assert [e.name for e in events] == expected_events\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"permissions,body,expected_status,expected_errors\",\n    (\n        ([\"create-table\"], {\"table\": \"t\", \"columns\": [{\"name\": \"c\"}]}, 201, None),\n        # Need insert-row too if you use \"rows\":\n        (\n            [\"create-table\"],\n            {\"table\": \"t\", \"rows\": [{\"name\": \"c\"}]},\n            403,\n            [\"Permission denied: need insert-row\"],\n        ),\n        # This should work:\n        (\n            [\"create-table\", \"insert-row\"],\n            {\"table\": \"t\", \"rows\": [{\"name\": \"c\"}]},\n            201,\n            None,\n        ),\n        # If you use replace: true you need update-row too:\n        (\n            [\"create-table\", \"insert-row\"],\n            {\"table\": \"t\", \"rows\": [{\"id\": 1}], \"pk\": \"id\", \"replace\": True},\n            403,\n            [\"Permission denied: need update-row\"],\n        ),\n    ),\n)\nasync def test_create_table_permissions(\n    ds_write, permissions, body, expected_status, expected_errors\n):\n    from datasette.tokens import TokenRestrictions\n\n    restrictions = TokenRestrictions()\n    for action in [\"view-instance\"] + permissions:\n        restrictions.allow_all(action)\n    token = await ds_write.create_token(\n        \"root\", handler=\"signed\", restrictions=restrictions\n    )\n    response = await ds_write.client.post(\n        \"/data/-/create\",\n        json=body,\n        headers=_headers(token),\n    )\n    assert response.status_code == expected_status\n    if expected_errors:\n        data = response.json()\n        assert data[\"ok\"] is False\n        assert data[\"errors\"] == expected_errors\n\n\n@pytest.mark.asyncio\n@pytest.mark.xfail(reason=\"Flaky, see https://github.com/simonw/datasette/issues/2356\")\n@pytest.mark.parametrize(\n    \"input,expected_rows_after\",\n    (\n        (\n            {\n                \"table\": \"test_insert_replace\",\n                \"rows\": [\n                    {\"id\": 1, \"name\": \"Row 1 new\"},\n                    {\"id\": 3, \"name\": \"Row 3 new\"},\n                ],\n                \"pk\": \"id\",\n                \"ignore\": True,\n            },\n            [\n                {\"id\": 1, \"name\": \"Row 1\"},\n                {\"id\": 2, \"name\": \"Row 2\"},\n                {\"id\": 3, \"name\": \"Row 3 new\"},\n            ],\n        ),\n        (\n            {\n                \"table\": \"test_insert_replace\",\n                \"rows\": [\n                    {\"id\": 1, \"name\": \"Row 1 new\"},\n                    {\"id\": 3, \"name\": \"Row 3 new\"},\n                ],\n                \"pk\": \"id\",\n                \"replace\": True,\n            },\n            [\n                {\"id\": 1, \"name\": \"Row 1 new\"},\n                {\"id\": 2, \"name\": \"Row 2\"},\n                {\"id\": 3, \"name\": \"Row 3 new\"},\n            ],\n        ),\n    ),\n)\nasync def test_create_table_ignore_replace(ds_write, input, expected_rows_after):\n    # Create table with two rows\n    token = write_token(ds_write)\n    first_response = await ds_write.client.post(\n        \"/data/-/create\",\n        json={\n            \"rows\": [{\"id\": 1, \"name\": \"Row 1\"}, {\"id\": 2, \"name\": \"Row 2\"}],\n            \"table\": \"test_insert_replace\",\n            \"pk\": \"id\",\n        },\n        headers=_headers(token),\n    )\n    assert first_response.status_code == 201\n\n    ds_write._tracked_events = []\n\n    # Try a second time\n    second_response = await ds_write.client.post(\n        \"/data/-/create\",\n        json=input,\n        headers=_headers(token),\n    )\n    assert second_response.status_code == 201\n    # Check that the rows are as expected\n    rows = await ds_write.client.get(\"/data/test_insert_replace.json?_shape=array\")\n    assert rows.json() == expected_rows_after\n\n    # Check it fired the right events\n    event_names = [e.name for e in ds_write._tracked_events]\n    assert event_names == [\"insert-rows\"]\n\n\n@pytest.mark.asyncio\nasync def test_create_table_error_if_pk_changed(ds_write):\n    token = write_token(ds_write)\n    first_response = await ds_write.client.post(\n        \"/data/-/create\",\n        json={\n            \"rows\": [{\"id\": 1, \"name\": \"Row 1\"}, {\"id\": 2, \"name\": \"Row 2\"}],\n            \"table\": \"test_insert_replace\",\n            \"pk\": \"id\",\n        },\n        headers=_headers(token),\n    )\n    assert first_response.status_code == 201\n    # Try a second time with a different pk\n    second_response = await ds_write.client.post(\n        \"/data/-/create\",\n        json={\n            \"rows\": [{\"id\": 1, \"name\": \"Row 1\"}, {\"id\": 2, \"name\": \"Row 2\"}],\n            \"table\": \"test_insert_replace\",\n            \"pk\": \"name\",\n            \"replace\": True,\n        },\n        headers=_headers(token),\n    )\n    assert second_response.status_code == 400\n    assert second_response.json() == {\n        \"ok\": False,\n        \"errors\": [\"pk cannot be changed for existing table\"],\n    }\n\n\n@pytest.mark.asyncio\nasync def test_create_table_error_rows_twice_with_duplicates(ds_write):\n    # Error if you don't send ignore: True or replace: True\n    token = write_token(ds_write)\n    input = {\n        \"rows\": [{\"id\": 1, \"name\": \"Row 1\"}, {\"id\": 2, \"name\": \"Row 2\"}],\n        \"table\": \"test_create_twice\",\n        \"pk\": \"id\",\n    }\n    first_response = await ds_write.client.post(\n        \"/data/-/create\",\n        json=input,\n        headers=_headers(token),\n    )\n    assert first_response.status_code == 201\n    second_response = await ds_write.client.post(\n        \"/data/-/create\",\n        json=input,\n        headers=_headers(token),\n    )\n    assert second_response.status_code == 400\n    assert second_response.json() == {\n        \"ok\": False,\n        \"errors\": [\"UNIQUE constraint failed: test_create_twice.id\"],\n    }\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"path\",\n    (\n        \"/data/-/create\",\n        \"/data/docs/-/drop\",\n        \"/data/docs/-/insert\",\n    ),\n)\nasync def test_method_not_allowed(ds_write, path):\n    response = await ds_write.client.get(\n        path,\n        headers={\n            \"Content-Type\": \"application/json\",\n        },\n    )\n    assert response.status_code == 405\n    assert response.json() == {\n        \"ok\": False,\n        \"error\": \"Method not allowed\",\n    }\n\n\n@pytest.mark.asyncio\nasync def test_create_uses_alter_by_default_for_new_table(ds_write):\n    ds_write._tracked_events = []\n    token = write_token(ds_write)\n    response = await ds_write.client.post(\n        \"/data/-/create\",\n        json={\n            \"table\": \"new_table\",\n            \"rows\": [\n                {\n                    \"name\": \"Row 1\",\n                }\n            ]\n            * 100\n            + [\n                {\"name\": \"Row 2\", \"extra\": \"Extra\"},\n            ],\n            \"pk\": \"id\",\n        },\n        headers=_headers(token),\n    )\n    assert response.status_code == 201\n    event_names = [e.name for e in ds_write._tracked_events]\n    assert event_names == [\"create-table\", \"insert-rows\"]\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\"has_alter_permission\", (True, False))\nasync def test_create_using_alter_against_existing_table(\n    ds_write, has_alter_permission\n):\n    token = write_token(\n        ds_write, permissions=[\"ir\", \"ct\"] + ([\"at\"] if has_alter_permission else [])\n    )\n    # First create the table\n    response = await ds_write.client.post(\n        \"/data/-/create\",\n        json={\n            \"table\": \"new_table\",\n            \"rows\": [\n                {\n                    \"name\": \"Row 1\",\n                }\n            ],\n            \"pk\": \"id\",\n        },\n        headers=_headers(token),\n    )\n    assert response.status_code == 201\n\n    ds_write._tracked_events = []\n    # Now try to insert more rows using /-/create with alter=True\n    response2 = await ds_write.client.post(\n        \"/data/-/create\",\n        json={\n            \"table\": \"new_table\",\n            \"rows\": [{\"name\": \"Row 2\", \"extra\": \"extra\"}],\n            \"pk\": \"id\",\n            \"alter\": True,\n        },\n        headers=_headers(token),\n    )\n    if not has_alter_permission:\n        assert response2.status_code == 403\n        assert response2.json() == {\n            \"ok\": False,\n            \"errors\": [\"Permission denied: need alter-table\"],\n        }\n    else:\n        assert response2.status_code == 201\n\n        event_names = [e.name for e in ds_write._tracked_events]\n        assert event_names == [\"alter-table\", \"insert-rows\"]\n\n        # It should have altered the table\n        alter_event = ds_write._tracked_events[0]\n        assert alter_event.name == \"alter-table\"\n        assert \"extra\" not in alter_event.before_schema\n        assert \"extra\" in alter_event.after_schema\n\n        insert_rows_event = ds_write._tracked_events[1]\n        assert insert_rows_event.name == \"insert-rows\"\n        assert insert_rows_event.num_rows == 1\n"
  },
  {
    "path": "tests/test_auth.py",
    "content": "from bs4 import BeautifulSoup as Soup\nfrom .utils import cookie_was_deleted, last_event\nfrom click.testing import CliRunner\nfrom datasette.utils import baseconv\nfrom datasette.cli import cli\nfrom datasette.resources import (\n    DatabaseResource,\n    TableResource,\n)\nimport pytest\nimport time\n\n\n@pytest.mark.asyncio\nasync def test_auth_token(ds_client):\n    \"\"\"The /-/auth-token endpoint sets the correct cookie\"\"\"\n    assert ds_client.ds._root_token is not None\n    path = f\"/-/auth-token?token={ds_client.ds._root_token}\"\n    response = await ds_client.get(path)\n    assert response.status_code == 302\n    assert \"/\" == response.headers[\"Location\"]\n    assert {\"a\": {\"id\": \"root\"}} == ds_client.ds.unsign(\n        response.cookies[\"ds_actor\"], \"actor\"\n    )\n    # Should have recorded a login event\n    event = last_event(ds_client.ds)\n    assert event.name == \"login\"\n    assert event.actor == {\"id\": \"root\"}\n    # Check that a second with same token fails\n    assert ds_client.ds._root_token is None\n    assert (await ds_client.get(path)).status_code == 403\n    # But attempting with same token while logged in as root should redirect to /\n    response = await ds_client.get(\n        path, cookies={\"ds_actor\": ds_client.actor_cookie({\"id\": \"root\"})}\n    )\n    assert response.status_code == 302\n    assert response.headers[\"Location\"] == \"/\"\n\n\n@pytest.mark.asyncio\nasync def test_actor_cookie(ds_client):\n    \"\"\"A valid actor cookie sets request.scope['actor']\"\"\"\n    cookie = ds_client.actor_cookie({\"id\": \"test\"})\n    await ds_client.get(\"/\", cookies={\"ds_actor\": cookie})\n    assert ds_client.ds._last_request.scope[\"actor\"] == {\"id\": \"test\"}\n\n\n@pytest.mark.asyncio\nasync def test_actor_cookie_invalid(ds_client):\n    cookie = ds_client.actor_cookie({\"id\": \"test\"})\n    # Break the signature\n    await ds_client.get(\"/\", cookies={\"ds_actor\": cookie[:-1] + \".\"})\n    assert ds_client.ds._last_request.scope[\"actor\"] is None\n    # Break the cookie format\n    cookie = ds_client.ds.sign({\"b\": {\"id\": \"test\"}}, \"actor\")\n    await ds_client.get(\"/\", cookies={\"ds_actor\": cookie})\n    assert ds_client.ds._last_request.scope[\"actor\"] is None\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"offset,expected\",\n    [\n        ((24 * 60 * 60), {\"id\": \"test\"}),\n        (-(24 * 60 * 60), None),\n    ],\n)\nasync def test_actor_cookie_that_expires(ds_client, offset, expected):\n    expires_at = int(time.time()) + offset\n    cookie = ds_client.ds.sign(\n        {\"a\": {\"id\": \"test\"}, \"e\": baseconv.base62.encode(expires_at)}, \"actor\"\n    )\n    await ds_client.get(\"/\", cookies={\"ds_actor\": cookie})\n    assert ds_client.ds._last_request.scope[\"actor\"] == expected\n\n\ndef test_logout(app_client):\n    # Keeping app_client for the moment because of csrftoken_from\n    response = app_client.get(\n        \"/-/logout\", cookies={\"ds_actor\": app_client.actor_cookie({\"id\": \"test\"})}\n    )\n    assert 200 == response.status\n    assert \"
    You are logged in as test
    \" in response.text\n    # Actors without an id get full serialization\n    response2 = app_client.get(\n        \"/-/logout\", cookies={\"ds_actor\": app_client.actor_cookie({\"name2\": \"bob\"})}\n    )\n    assert 200 == response2.status\n    assert (\n        \"
    You are logged in as {'name2': 'bob'}
    \"\n        in response2.text\n    )\n    # If logged out you get a redirect to /\n    response3 = app_client.get(\"/-/logout\")\n    assert 302 == response3.status\n    # A POST to that page should log the user out\n    response4 = app_client.post(\n        \"/-/logout\",\n        csrftoken_from=True,\n        cookies={\"ds_actor\": app_client.actor_cookie({\"id\": \"test\"})},\n    )\n    # Should have recorded a logout event\n    event = last_event(app_client.ds)\n    assert event.name == \"logout\"\n    assert event.actor == {\"id\": \"test\"}\n    # The ds_actor cookie should have been unset\n    assert cookie_was_deleted(response4, \"ds_actor\")\n    # Should also have set a message\n    messages = app_client.ds.unsign(response4.cookies[\"ds_messages\"], \"messages\")\n    assert [[\"You are now logged out\", 2]] == messages\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\"path\", [\"/\", \"/fixtures\", \"/fixtures/facetable\"])\nasync def test_logout_button_in_navigation(ds_client, path):\n    response = await ds_client.get(\n        path, cookies={\"ds_actor\": ds_client.actor_cookie({\"id\": \"test\"})}\n    )\n    anon_response = await ds_client.get(path)\n    for fragment in (\n        \"test\",\n        '',\n    ):\n        assert fragment in response.text\n        assert fragment not in anon_response.text\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\"path\", [\"/\", \"/fixtures\", \"/fixtures/facetable\"])\nasync def test_no_logout_button_in_navigation_if_no_ds_actor_cookie(ds_client, path):\n    response = await ds_client.get(path + \"?_bot=1\")\n    assert \"bot\" in response.text\n    assert (\n        ''\n        not in response.text\n    )\n\n\n@pytest.mark.parametrize(\n    \"post_data,errors,expected_duration,expected_r\",\n    (\n        ({\"expire_type\": \"\"}, [], None, None),\n        ({\"expire_type\": \"x\"}, [\"Invalid expire duration\"], None, None),\n        ({\"expire_type\": \"minutes\"}, [\"Invalid expire duration\"], None, None),\n        (\n            {\"expire_type\": \"minutes\", \"expire_duration\": \"x\"},\n            [\"Invalid expire duration\"],\n            None,\n            None,\n        ),\n        (\n            {\"expire_type\": \"minutes\", \"expire_duration\": \"-1\"},\n            [\"Invalid expire duration\"],\n            None,\n            None,\n        ),\n        (\n            {\"expire_type\": \"minutes\", \"expire_duration\": \"0\"},\n            [\"Invalid expire duration\"],\n            None,\n            None,\n        ),\n        ({\"expire_type\": \"minutes\", \"expire_duration\": \"10\"}, [], 600, None),\n        ({\"expire_type\": \"hours\", \"expire_duration\": \"10\"}, [], 10 * 60 * 60, None),\n        ({\"expire_type\": \"days\", \"expire_duration\": \"3\"}, [], 60 * 60 * 24 * 3, None),\n        # Token restrictions\n        ({\"all:view-instance\": \"on\"}, [], None, {\"a\": [\"vi\"]}),\n        ({\"database:fixtures:view-query\": \"on\"}, [], None, {\"d\": {\"fixtures\": [\"vq\"]}}),\n        (\n            {\"resource:fixtures:facetable:insert-row\": \"on\"},\n            [],\n            None,\n            {\"r\": {\"fixtures\": {\"facetable\": [\"ir\"]}}},\n        ),\n    ),\n)\ndef test_auth_create_token(\n    app_client, post_data, errors, expected_duration, expected_r\n):\n    assert app_client.get(\"/-/create-token\").status == 403\n    ds_actor = app_client.actor_cookie({\"id\": \"test\"})\n    response = app_client.get(\"/-/create-token\", cookies={\"ds_actor\": ds_actor})\n    assert response.status == 200\n    assert \">Create an API token<\" in response.text\n    # Confirm some aspects of expected set of checkboxes\n    soup = Soup(response.text, \"html.parser\")\n    checkbox_names = {el[\"name\"] for el in soup.select('input[type=\"checkbox\"]')}\n    assert checkbox_names.issuperset(\n        {\n            \"all:view-instance\",\n            \"all:view-query\",\n            \"database:fixtures:drop-table\",\n            \"resource:fixtures:foreign_key_references:insert-row\",\n            \"resource:fixtures:facetable:set-column-type\",\n        }\n    )\n    # Now try actually creating one\n    response2 = app_client.post(\n        \"/-/create-token\",\n        post_data,\n        csrftoken_from=True,\n        cookies={\"ds_actor\": ds_actor},\n    )\n    assert response2.status == 200\n    if errors:\n        for error in errors:\n            assert '
    {}
    '.format(error) in response2.text\n    else:\n        # Check create-token event\n        event = last_event(app_client.ds)\n        assert event.name == \"create-token\"\n        assert event.expires_after == expected_duration\n        assert isinstance(event.restrict_all, list)\n        assert isinstance(event.restrict_database, dict)\n        assert isinstance(event.restrict_resource, dict)\n        # Extract token from page\n        token = response2.text.split('value=\"dstok_')[1].split('\"')[0]\n        details = app_client.ds.unsign(token, \"token\")\n        if expected_r:\n            r = details.pop(\"_r\")\n            assert r == expected_r\n        assert details.keys() == {\"a\", \"t\", \"d\"} or details.keys() == {\"a\", \"t\"}\n        assert details[\"a\"] == \"test\"\n        if expected_duration is None:\n            assert \"d\" not in details\n        else:\n            assert details[\"d\"] == expected_duration\n        # And test that token\n        response3 = app_client.get(\n            \"/-/actor.json\",\n            headers={\"Authorization\": \"Bearer {}\".format(\"dstok_{}\".format(token))},\n        )\n        assert response3.status == 200\n        assert response3.json[\"actor\"][\"id\"] == \"test\"\n\n\n@pytest.mark.asyncio\nasync def test_auth_create_token_not_allowed_for_tokens(ds_client):\n    ds_tok = ds_client.ds.sign({\"a\": \"test\", \"token\": \"dstok\"}, \"token\")\n    response = await ds_client.get(\n        \"/-/create-token\",\n        headers={\"Authorization\": \"Bearer dstok_{}\".format(ds_tok)},\n    )\n    assert response.status_code == 403\n\n\n@pytest.mark.asyncio\nasync def test_auth_create_token_not_allowed_if_allow_signed_tokens_off(ds_client):\n    ds_client.ds._settings[\"allow_signed_tokens\"] = False\n    try:\n        ds_actor = ds_client.actor_cookie({\"id\": \"test\"})\n        response = await ds_client.get(\n            \"/-/create-token\", cookies={\"ds_actor\": ds_actor}\n        )\n        assert response.status_code == 403\n    finally:\n        ds_client.ds._settings[\"allow_signed_tokens\"] = True\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"scenario,should_work\",\n    (\n        (\"allow_signed_tokens_off\", False),\n        (\"no_token\", False),\n        (\"no_timestamp\", False),\n        (\"invalid_token\", False),\n        (\"expired_token\", False),\n        (\"valid_unlimited_token\", True),\n        (\"valid_expiring_token\", True),\n    ),\n)\nasync def test_auth_with_dstok_token(ds_client, scenario, should_work):\n    token = None\n    _time = int(time.time())\n    if scenario in (\"valid_unlimited_token\", \"allow_signed_tokens_off\"):\n        token = ds_client.ds.sign({\"a\": \"test\", \"t\": _time}, \"token\")\n    elif scenario == \"valid_expiring_token\":\n        token = ds_client.ds.sign({\"a\": \"test\", \"t\": _time - 50, \"d\": 1000}, \"token\")\n    elif scenario == \"expired_token\":\n        token = ds_client.ds.sign({\"a\": \"test\", \"t\": _time - 2000, \"d\": 1000}, \"token\")\n    elif scenario == \"no_timestamp\":\n        token = ds_client.ds.sign({\"a\": \"test\"}, \"token\")\n    elif scenario == \"invalid_token\":\n        token = \"invalid\"\n    if token:\n        token = \"dstok_{}\".format(token)\n    if scenario == \"allow_signed_tokens_off\":\n        ds_client.ds._settings[\"allow_signed_tokens\"] = False\n    headers = {}\n    if token:\n        headers[\"Authorization\"] = \"Bearer {}\".format(token)\n    response = await ds_client.get(\"/-/actor.json\", headers=headers)\n    try:\n        if should_work:\n            data = response.json()\n            assert data.keys() == {\"actor\"}\n            actor = data[\"actor\"]\n            expected_keys = {\"id\", \"token\"}\n            if scenario != \"valid_unlimited_token\":\n                expected_keys.add(\"token_expires\")\n            assert actor.keys() == expected_keys\n            assert actor[\"id\"] == \"test\"\n            assert actor[\"token\"] == \"dstok\"\n            if scenario != \"valid_unlimited_token\":\n                assert isinstance(actor[\"token_expires\"], int)\n        else:\n            assert response.json() == {\"actor\": None}\n    finally:\n        ds_client.ds._settings[\"allow_signed_tokens\"] = True\n\n\n@pytest.mark.parametrize(\"expires\", (None, 1000, -1000))\ndef test_cli_create_token(app_client, expires):\n    secret = app_client.ds._secret\n    runner = CliRunner()\n    args = [\"create-token\", \"--secret\", secret, \"test\"]\n    if expires:\n        args += [\"--expires-after\", str(expires)]\n    result = runner.invoke(cli, args)\n    assert result.exit_code == 0\n    token = result.output.strip()\n    assert token.startswith(\"dstok_\")\n    details = app_client.ds.unsign(token[len(\"dstok_\") :], \"token\")\n    expected_keys = {\"a\", \"t\"}\n    if expires:\n        expected_keys.add(\"d\")\n    assert details.keys() == expected_keys\n    assert details[\"a\"] == \"test\"\n    response = app_client.get(\n        \"/-/actor.json\", headers={\"Authorization\": \"Bearer {}\".format(token)}\n    )\n    if expires is None or expires > 0:\n        expected_actor = {\n            \"id\": \"test\",\n            \"token\": \"dstok\",\n        }\n        if expires and expires > 0:\n            expected_actor[\"token_expires\"] = details[\"t\"] + expires\n        assert response.json == {\"actor\": expected_actor}\n    else:\n        expected_actor = None\n    assert response.json == {\"actor\": expected_actor}\n\n\n@pytest.mark.asyncio\nasync def test_root_with_root_enabled_gets_all_permissions(ds_client):\n    \"\"\"Root user with root_enabled=True gets all permissions\"\"\"\n    # Ensure catalog tables are populated\n    await ds_client.ds.invoke_startup()\n    await ds_client.ds._refresh_schemas()\n\n    # Set root_enabled to simulate --root flag\n    ds_client.ds.root_enabled = True\n\n    root_actor = {\"id\": \"root\"}\n\n    # Test instance-level permissions (no resource)\n    assert (\n        await ds_client.ds.allowed(action=\"permissions-debug\", actor=root_actor) is True\n    )\n    assert await ds_client.ds.allowed(action=\"debug-menu\", actor=root_actor) is True\n\n    # Test view permissions using the new ds.allowed() method\n    assert await ds_client.ds.allowed(action=\"view-instance\", actor=root_actor) is True\n\n    assert (\n        await ds_client.ds.allowed(\n            action=\"view-database\",\n            resource=DatabaseResource(\"fixtures\"),\n            actor=root_actor,\n        )\n        is True\n    )\n\n    assert (\n        await ds_client.ds.allowed(\n            action=\"view-table\",\n            resource=TableResource(\"fixtures\", \"facetable\"),\n            actor=root_actor,\n        )\n        is True\n    )\n\n    # Test write permissions using ds.allowed()\n    assert (\n        await ds_client.ds.allowed(\n            action=\"insert-row\",\n            resource=TableResource(\"fixtures\", \"facetable\"),\n            actor=root_actor,\n        )\n        is True\n    )\n\n    assert (\n        await ds_client.ds.allowed(\n            action=\"delete-row\",\n            resource=TableResource(\"fixtures\", \"facetable\"),\n            actor=root_actor,\n        )\n        is True\n    )\n\n    assert (\n        await ds_client.ds.allowed(\n            action=\"update-row\",\n            resource=TableResource(\"fixtures\", \"facetable\"),\n            actor=root_actor,\n        )\n        is True\n    )\n\n    assert (\n        await ds_client.ds.allowed(\n            action=\"create-table\",\n            resource=DatabaseResource(\"fixtures\"),\n            actor=root_actor,\n        )\n        is True\n    )\n\n    assert (\n        await ds_client.ds.allowed(\n            action=\"alter-table\",\n            resource=TableResource(\"fixtures\", \"facetable\"),\n            actor=root_actor,\n        )\n        is True\n    )\n\n    assert (\n        await ds_client.ds.allowed(\n            action=\"set-column-type\",\n            resource=TableResource(\"fixtures\", \"facetable\"),\n            actor=root_actor,\n        )\n        is True\n    )\n\n    assert (\n        await ds_client.ds.allowed(\n            action=\"drop-table\",\n            resource=TableResource(\"fixtures\", \"facetable\"),\n            actor=root_actor,\n        )\n        is True\n    )\n\n\n@pytest.mark.asyncio\nasync def test_root_without_root_enabled_no_special_permissions(ds_client):\n    \"\"\"Root user without root_enabled doesn't get automatic permissions\"\"\"\n    # Ensure catalog tables are populated\n    await ds_client.ds.invoke_startup()\n    await ds_client.ds._refresh_schemas()\n\n    # Ensure root_enabled is NOT set (or is False)\n    ds_client.ds.root_enabled = False\n\n    root_actor = {\"id\": \"root\"}\n\n    # Test permissions that normally require special access\n    # Without root_enabled, root should follow normal permission rules\n\n    # View permissions should still work (default=True)\n    assert (\n        await ds_client.ds.allowed(action=\"view-instance\", actor=root_actor) is True\n    )  # Default permission\n\n    assert (\n        await ds_client.ds.allowed(\n            action=\"view-database\",\n            resource=DatabaseResource(\"fixtures\"),\n            actor=root_actor,\n        )\n        is True\n    )  # Default permission\n\n    # But restricted permissions should NOT automatically be granted\n    # Test with instance-level permission (no resource class)\n    result = await ds_client.ds.allowed(action=\"permissions-debug\", actor=root_actor)\n    assert (\n        result is not True\n    ), \"Root without root_enabled should not automatically get permissions-debug\"\n\n    # Test with resource-based permissions using ds.allowed()\n    assert (\n        await ds_client.ds.allowed(\n            action=\"create-table\",\n            resource=DatabaseResource(\"fixtures\"),\n            actor=root_actor,\n        )\n        is not True\n    ), \"Root without root_enabled should not automatically get create-table\"\n\n    assert (\n        await ds_client.ds.allowed(\n            action=\"drop-table\",\n            resource=TableResource(\"fixtures\", \"facetable\"),\n            actor=root_actor,\n        )\n        is not True\n    ), \"Root without root_enabled should not automatically get drop-table\"\n\n    assert (\n        await ds_client.ds.allowed(\n            action=\"set-column-type\",\n            resource=TableResource(\"fixtures\", \"facetable\"),\n            actor=root_actor,\n        )\n        is not True\n    ), \"Root without root_enabled should not automatically get set-column-type\"\n"
  },
  {
    "path": "tests/test_base_view.py",
    "content": "from datasette.views.base import View\nfrom datasette import Request, Response\nfrom datasette.app import Datasette\nimport json\nimport pytest\n\n\nclass GetView(View):\n    async def get(self, request, datasette):\n        return Response.json(\n            {\n                \"absolute_url\": datasette.absolute_url(request, \"/\"),\n                \"request_path\": request.path,\n            }\n        )\n\n\nclass GetAndPostView(GetView):\n    async def post(self, request, datasette):\n        return Response.json(\n            {\n                \"method\": request.method,\n                \"absolute_url\": datasette.absolute_url(request, \"/\"),\n                \"request_path\": request.path,\n            }\n        )\n\n\n@pytest.mark.asyncio\nasync def test_get_view():\n    v = GetView()\n    datasette = Datasette()\n    response = await v(Request.fake(\"/foo\"), datasette)\n    assert json.loads(response.body) == {\n        \"absolute_url\": \"http://localhost/\",\n        \"request_path\": \"/foo\",\n    }\n    # Try a HEAD request\n    head_response = await v(Request.fake(\"/foo\", method=\"HEAD\"), datasette)\n    assert head_response.body == \"\"\n    assert head_response.status == 200\n    # And OPTIONS\n    options_response = await v(Request.fake(\"/foo\", method=\"OPTIONS\"), datasette)\n    assert options_response.body == \"ok\"\n    assert options_response.status == 200\n    assert options_response.headers[\"allow\"] == \"HEAD, GET\"\n    # And POST\n    post_response = await v(Request.fake(\"/foo\", method=\"POST\"), datasette)\n    assert post_response.body == \"Method not allowed\"\n    assert post_response.status == 405\n    # And POST with .json extension\n    post_json_response = await v(Request.fake(\"/foo.json\", method=\"POST\"), datasette)\n    assert json.loads(post_json_response.body) == {\n        \"ok\": False,\n        \"error\": \"Method not allowed\",\n    }\n    assert post_json_response.status == 405\n\n\n@pytest.mark.asyncio\nasync def test_post_view():\n    v = GetAndPostView()\n    datasette = Datasette()\n    response = await v(Request.fake(\"/foo\"), datasette)\n    assert json.loads(response.body) == {\n        \"absolute_url\": \"http://localhost/\",\n        \"request_path\": \"/foo\",\n    }\n    # Try a HEAD request\n    head_response = await v(Request.fake(\"/foo\", method=\"HEAD\"), datasette)\n    assert head_response.body == \"\"\n    assert head_response.status == 200\n    # And OPTIONS\n    options_response = await v(Request.fake(\"/foo\", method=\"OPTIONS\"), datasette)\n    assert options_response.body == \"ok\"\n    assert options_response.status == 200\n    assert options_response.headers[\"allow\"] == \"HEAD, GET, POST\"\n    # And POST\n    post_response = await v(Request.fake(\"/foo\", method=\"POST\"), datasette)\n    assert json.loads(post_response.body) == {\n        \"method\": \"POST\",\n        \"absolute_url\": \"http://localhost/\",\n        \"request_path\": \"/foo\",\n    }\n"
  },
  {
    "path": "tests/test_canned_queries.py",
    "content": "from bs4 import BeautifulSoup as Soup\nimport json\nimport pytest\nimport re\nfrom .fixtures import make_app_client\n\n\n@pytest.fixture\ndef canned_write_client(tmpdir):\n    template_dir = tmpdir / \"canned_write_templates\"\n    template_dir.mkdir()\n    (template_dir / \"query-data-update_name.html\").write_text(\n        \"\"\"\n    {% extends \"query.html\" %}\n    {% block content %}!!!CUSTOM_UPDATE_NAME_TEMPLATE!!!{{ super() }}{% endblock %}\n    \"\"\",\n        \"utf-8\",\n    )\n    with make_app_client(\n        extra_databases={\"data.db\": \"create table names (name text)\"},\n        template_dir=str(template_dir),\n        config={\n            \"databases\": {\n                \"data\": {\n                    \"queries\": {\n                        \"canned_read\": {\"sql\": \"select * from names\"},\n                        \"add_name\": {\n                            \"sql\": \"insert into names (name) values (:name)\",\n                            \"write\": True,\n                            \"on_success_redirect\": \"/data/add_name?success\",\n                        },\n                        \"add_name_specify_id\": {\n                            \"sql\": \"insert into names (rowid, name) values (:rowid, :name)\",\n                            \"on_success_message_sql\": \"select 'Name added: ' || :name || ' with rowid ' || :rowid\",\n                            \"write\": True,\n                            \"on_error_redirect\": \"/data/add_name_specify_id?error\",\n                        },\n                        \"add_name_specify_id_with_error_in_on_success_message_sql\": {\n                            \"sql\": \"insert into names (rowid, name) values (:rowid, :name)\",\n                            \"on_success_message_sql\": \"select this is bad SQL\",\n                            \"write\": True,\n                        },\n                        \"delete_name\": {\n                            \"sql\": \"delete from names where rowid = :rowid\",\n                            \"write\": True,\n                            \"on_success_message\": \"Name deleted\",\n                            \"allow\": {\"id\": \"root\"},\n                        },\n                        \"update_name\": {\n                            \"sql\": \"update names set name = :name where rowid = :rowid\",\n                            \"params\": [\"rowid\", \"name\", \"extra\"],\n                            \"write\": True,\n                        },\n                    }\n                }\n            }\n        },\n    ) as client:\n        yield client\n\n\n@pytest.fixture\ndef canned_write_immutable_client():\n    with make_app_client(\n        is_immutable=True,\n        config={\n            \"databases\": {\n                \"fixtures\": {\n                    \"queries\": {\n                        \"add\": {\n                            \"sql\": \"insert into sortable (text) values (:text)\",\n                            \"write\": True,\n                        },\n                    }\n                }\n            }\n        },\n    ) as client:\n        yield client\n\n\n@pytest.mark.asyncio\nasync def test_canned_query_with_named_parameter(ds_client):\n    response = await ds_client.get(\n        \"/fixtures/neighborhood_search.json?text=town&_shape=arrays\"\n    )\n    assert response.json()[\"rows\"] == [\n        [\"Corktown\", \"Detroit\", \"MI\"],\n        [\"Downtown\", \"Los Angeles\", \"CA\"],\n        [\"Downtown\", \"Detroit\", \"MI\"],\n        [\"Greektown\", \"Detroit\", \"MI\"],\n        [\"Koreatown\", \"Los Angeles\", \"CA\"],\n        [\"Mexicantown\", \"Detroit\", \"MI\"],\n    ]\n\n\ndef test_insert(canned_write_client):\n    response = canned_write_client.post(\n        \"/data/add_name\",\n        {\"name\": \"Hello\"},\n        csrftoken_from=True,\n        cookies={\"foo\": \"bar\"},\n    )\n    messages = canned_write_client.ds.unsign(\n        response.cookies[\"ds_messages\"], \"messages\"\n    )\n    assert messages == [[\"Query executed, 1 row affected\", 1]]\n    assert response.status == 302\n    assert response.headers[\"Location\"] == \"/data/add_name?success\"\n\n\n@pytest.mark.parametrize(\n    \"query_name,expect_csrf_hidden_field\",\n    [\n        (\"canned_read\", False),\n        (\"add_name_specify_id\", True),\n        (\"add_name\", True),\n    ],\n)\ndef test_canned_query_form_csrf_hidden_field(\n    canned_write_client, query_name, expect_csrf_hidden_field\n):\n    response = canned_write_client.get(f\"/data/{query_name}\")\n    html = response.text\n    fragment = '' in response.text\n\n\ndef test_vary_header(canned_write_client):\n    # These forms embed a csrftoken so they should be served with Vary: Cookie\n    assert \"vary\" not in canned_write_client.get(\"/data\").headers\n    assert \"Cookie\" == canned_write_client.get(\"/data/update_name\").headers[\"vary\"]\n\n\ndef test_json_post_body(canned_write_client):\n    response = canned_write_client.post(\n        \"/data/add_name\",\n        body=json.dumps({\"name\": [\"Hello\", \"there\"]}),\n    )\n    assert 302 == response.status\n    assert \"/data/add_name?success\" == response.headers[\"Location\"]\n    rows = canned_write_client.get(\"/data/names.json?_shape=array\").json\n    assert rows == [{\"rowid\": 1, \"name\": \"['Hello', 'there']\"}]\n\n\n@pytest.mark.parametrize(\n    \"headers,body,querystring\",\n    (\n        (None, \"name=NameGoesHere\", \"?_json=1\"),\n        ({\"Accept\": \"application/json\"}, \"name=NameGoesHere\", None),\n        (None, \"name=NameGoesHere&_json=1\", None),\n        (None, '{\"name\": \"NameGoesHere\", \"_json\": 1}', None),\n    ),\n)\ndef test_json_response(canned_write_client, headers, body, querystring):\n    response = canned_write_client.post(\n        \"/data/add_name\" + (querystring or \"\"),\n        body=body,\n        headers=headers,\n    )\n    assert 200 == response.status\n    assert response.headers[\"content-type\"] == \"application/json; charset=utf-8\"\n    assert response.json == {\n        \"ok\": True,\n        \"message\": \"Query executed, 1 row affected\",\n        \"redirect\": \"/data/add_name?success\",\n    }\n    rows = canned_write_client.get(\"/data/names.json?_shape=array\").json\n    assert rows == [{\"rowid\": 1, \"name\": \"NameGoesHere\"}]\n\n\ndef test_canned_query_permissions_on_database_page(canned_write_client):\n    # Without auth only shows three queries\n    query_names = {\n        q[\"name\"] for q in canned_write_client.get(\"/data.json\").json[\"queries\"]\n    }\n    assert query_names == {\n        \"add_name_specify_id_with_error_in_on_success_message_sql\",\n        \"from_hook\",\n        \"update_name\",\n        \"add_name_specify_id\",\n        \"from_async_hook\",\n        \"canned_read\",\n        \"add_name\",\n    }\n\n    # With auth shows four\n    response = canned_write_client.get(\n        \"/data.json\",\n        cookies={\"ds_actor\": canned_write_client.actor_cookie({\"id\": \"root\"})},\n    )\n    assert response.status == 200\n    query_names_and_private = sorted(\n        [\n            {\"name\": q[\"name\"], \"private\": q[\"private\"]}\n            for q in response.json[\"queries\"]\n        ],\n        key=lambda q: q[\"name\"],\n    )\n    assert query_names_and_private == [\n        {\"name\": \"add_name\", \"private\": False},\n        {\"name\": \"add_name_specify_id\", \"private\": False},\n        {\n            \"name\": \"add_name_specify_id_with_error_in_on_success_message_sql\",\n            \"private\": False,\n        },\n        {\"name\": \"canned_read\", \"private\": False},\n        {\"name\": \"delete_name\", \"private\": True},\n        {\"name\": \"from_async_hook\", \"private\": False},\n        {\"name\": \"from_hook\", \"private\": False},\n        {\"name\": \"update_name\", \"private\": False},\n    ]\n\n\ndef test_canned_query_permissions(canned_write_client):\n    assert 403 == canned_write_client.get(\"/data/delete_name\").status\n    assert 200 == canned_write_client.get(\"/data/update_name\").status\n    cookies = {\"ds_actor\": canned_write_client.actor_cookie({\"id\": \"root\"})}\n    assert 200 == canned_write_client.get(\"/data/delete_name\", cookies=cookies).status\n    assert 200 == canned_write_client.get(\"/data/update_name\", cookies=cookies).status\n\n\n@pytest.fixture(scope=\"session\")\ndef magic_parameters_client():\n    with make_app_client(\n        extra_databases={\"data.db\": \"create table logs (line text)\"},\n        config={\n            \"databases\": {\n                \"data\": {\n                    \"queries\": {\n                        \"runme_post\": {\"sql\": \"\", \"write\": True},\n                        \"runme_get\": {\"sql\": \"\"},\n                    }\n                }\n            }\n        },\n    ) as client:\n        yield client\n\n\n@pytest.mark.parametrize(\n    \"magic_parameter,expected_re\",\n    [\n        (\"_actor_id\", \"root\"),\n        (\"_header_host\", \"localhost\"),\n        (\"_header_not_a_thing\", \"\"),\n        (\"_cookie_foo\", \"bar\"),\n        (\"_now_epoch\", r\"^\\d+$\"),\n        (\"_now_date_utc\", r\"^\\d{4}-\\d{2}-\\d{2}$\"),\n        (\"_now_datetime_utc\", r\"^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z$\"),\n        (\"_random_chars_1\", r\"^\\w$\"),\n        (\"_random_chars_10\", r\"^\\w{10}$\"),\n    ],\n)\ndef test_magic_parameters(magic_parameters_client, magic_parameter, expected_re):\n    magic_parameters_client.ds.config[\"databases\"][\"data\"][\"queries\"][\"runme_post\"][\n        \"sql\"\n    ] = f\"insert into logs (line) values (:{magic_parameter})\"\n    magic_parameters_client.ds.config[\"databases\"][\"data\"][\"queries\"][\"runme_get\"][\n        \"sql\"\n    ] = f\"select :{magic_parameter} as result\"\n    cookies = {\n        \"ds_actor\": magic_parameters_client.actor_cookie({\"id\": \"root\"}),\n        \"foo\": \"bar\",\n    }\n    # Test the GET version\n    get_response = magic_parameters_client.get(\n        \"/data/runme_get.json?_shape=array\", cookies=cookies\n    )\n    get_actual = get_response.json[0][\"result\"]\n    assert re.match(expected_re, str(get_actual))\n    # Test the form\n    form_response = magic_parameters_client.get(\"/data/runme_post\")\n    soup = Soup(form_response.body, \"html.parser\")\n    # The magic parameter should not be represented as a form field\n    assert None is soup.find(\"input\", {\"name\": magic_parameter})\n    # Submit the form to create a log line\n    response = magic_parameters_client.post(\n        \"/data/runme_post?_json=1\", {}, csrftoken_from=True, cookies=cookies\n    )\n    assert response.json == {\n        \"ok\": True,\n        \"message\": \"Query executed, 1 row affected\",\n        \"redirect\": None,\n    }\n    post_actual = magic_parameters_client.get(\n        \"/data/logs.json?_sort_desc=rowid&_shape=array\"\n    ).json[0][\"line\"]\n    assert re.match(expected_re, post_actual)\n\n\n@pytest.mark.parametrize(\"use_csrf\", [True, False])\n@pytest.mark.parametrize(\"return_json\", [True, False])\ndef test_magic_parameters_csrf_json(magic_parameters_client, use_csrf, return_json):\n    magic_parameters_client.ds.config[\"databases\"][\"data\"][\"queries\"][\"runme_post\"][\n        \"sql\"\n    ] = \"insert into logs (line) values (:_header_host)\"\n    qs = \"\"\n    if return_json:\n        qs = \"?_json=1\"\n    response = magic_parameters_client.post(\n        f\"/data/runme_post{qs}\",\n        {},\n        csrftoken_from=use_csrf or None,\n    )\n    if return_json:\n        assert response.status == 200\n        assert response.json[\"ok\"], response.json\n    else:\n        assert response.status == 302\n        messages = magic_parameters_client.ds.unsign(\n            response.cookies[\"ds_messages\"], \"messages\"\n        )\n        assert [[\"Query executed, 1 row affected\", 1]] == messages\n    post_actual = magic_parameters_client.get(\n        \"/data/logs.json?_sort_desc=rowid&_shape=array\"\n    ).json[0][\"line\"]\n    assert post_actual == \"localhost\"\n\n\ndef test_magic_parameters_cannot_be_used_in_arbitrary_queries(magic_parameters_client):\n    response = magic_parameters_client.get(\n        \"/data/-/query.json?sql=select+:_header_host&_shape=array\"\n    )\n    assert 400 == response.status\n    assert response.json[\"error\"].startswith(\"You did not supply a value for binding\")\n\n\ndef test_canned_write_custom_template(canned_write_client):\n    response = canned_write_client.get(\"/data/update_name\")\n    assert response.status == 200\n    assert \"!!!CUSTOM_UPDATE_NAME_TEMPLATE!!!\" in response.text\n    assert (\n        \"\"\n        in response.text\n    )\n    # And test for link rel=alternate while we're here:\n    assert (\n        ''\n        in response.text\n    )\n    assert (\n        response.headers[\"link\"]\n        == '; rel=\"alternate\"; type=\"application/json+datasette\"'\n    )\n\n\ndef test_canned_write_query_disabled_for_immutable_database(\n    canned_write_immutable_client,\n):\n    response = canned_write_immutable_client.get(\"/fixtures/add\")\n    assert response.status == 200\n    assert (\n        \"This query cannot be executed because the database is immutable.\"\n        in response.text\n    )\n    assert '' in response.text\n    # Submitting form should get a forbidden error\n    response = canned_write_immutable_client.post(\n        \"/fixtures/add\",\n        {\"text\": \"text\"},\n        csrftoken_from=True,\n    )\n    assert response.status == 403\n    assert \"Database is immutable\" in response.text\n"
  },
  {
    "path": "tests/test_cli.py",
    "content": "from .fixtures import (\n    make_app_client,\n    TestClient as _TestClient,\n    EXPECTED_PLUGINS,\n)\nfrom datasette.app import SETTINGS\nfrom datasette.plugins import DEFAULT_PLUGINS, pm\nfrom datasette.cli import cli, serve\nfrom datasette.version import __version__\nfrom datasette.utils import tilde_encode\nfrom datasette.utils.sqlite import sqlite3\nfrom click.testing import CliRunner\nimport io\nimport json\nimport pathlib\nimport pytest\nimport sys\nimport textwrap\nfrom unittest import mock\n\n\ndef test_inspect_cli(app_client):\n    runner = CliRunner()\n    result = runner.invoke(cli, [\"inspect\", \"fixtures.db\"])\n    data = json.loads(result.output)\n    assert [\"fixtures\"] == list(data.keys())\n    database = data[\"fixtures\"]\n    assert \"fixtures.db\" == database[\"file\"]\n    assert isinstance(database[\"hash\"], str)\n    assert 64 == len(database[\"hash\"])\n    for table_name, expected_count in {\n        \"Table With Space In Name\": 0,\n        \"facetable\": 15,\n    }.items():\n        assert expected_count == database[\"tables\"][table_name][\"count\"]\n\n\ndef test_inspect_cli_writes_to_file(app_client):\n    runner = CliRunner()\n    result = runner.invoke(\n        cli, [\"inspect\", \"fixtures.db\", \"--inspect-file\", \"foo.json\"]\n    )\n    assert 0 == result.exit_code, result.output\n    with open(\"foo.json\") as fp:\n        data = json.load(fp)\n    assert [\"fixtures\"] == list(data.keys())\n\n\ndef test_serve_with_inspect_file_prepopulates_table_counts_cache():\n    inspect_data = {\"fixtures\": {\"tables\": {\"hithere\": {\"count\": 44}}}}\n    with make_app_client(inspect_data=inspect_data, is_immutable=True) as client:\n        assert inspect_data == client.ds.inspect_data\n        db = client.ds.databases[\"fixtures\"]\n        assert {\"hithere\": 44} == db.cached_table_counts\n\n\n@pytest.mark.parametrize(\n    \"spatialite_paths,should_suggest_load_extension\",\n    (\n        ([], False),\n        ([\"/tmp\"], True),\n    ),\n)\ndef test_spatialite_error_if_attempt_to_open_spatialite(\n    spatialite_paths, should_suggest_load_extension\n):\n    with mock.patch(\"datasette.utils.SPATIALITE_PATHS\", spatialite_paths):\n        runner = CliRunner()\n        result = runner.invoke(\n            cli, [\"serve\", str(pathlib.Path(__file__).parent / \"spatialite.db\")]\n        )\n        assert result.exit_code != 0\n        assert \"It looks like you're trying to load a SpatiaLite\" in result.output\n        suggestion = \"--load-extension=spatialite\"\n        if should_suggest_load_extension:\n            assert suggestion in result.output\n        else:\n            assert suggestion not in result.output\n\n\n@mock.patch(\"datasette.utils.SPATIALITE_PATHS\", [\"/does/not/exist\"])\ndef test_spatialite_error_if_cannot_find_load_extension_spatialite():\n    runner = CliRunner()\n    result = runner.invoke(\n        cli,\n        [\n            \"serve\",\n            str(pathlib.Path(__file__).parent / \"spatialite.db\"),\n            \"--load-extension\",\n            \"spatialite\",\n        ],\n    )\n    assert result.exit_code != 0\n    assert \"Could not find SpatiaLite extension\" in result.output\n\n\ndef test_plugins_cli(app_client):\n    runner = CliRunner()\n    result1 = runner.invoke(cli, [\"plugins\"])\n    actual_plugins = sorted(\n        [p for p in json.loads(result1.output) if p[\"name\"] != \"TrackEventPlugin\"],\n        key=lambda p: p[\"name\"],\n    )\n    assert actual_plugins == EXPECTED_PLUGINS\n    # Try with --all\n    result2 = runner.invoke(cli, [\"plugins\", \"--all\"])\n    names = [p[\"name\"] for p in json.loads(result2.output)]\n    # Should have all the EXPECTED_PLUGINS\n    assert set(names).issuperset({p[\"name\"] for p in EXPECTED_PLUGINS})\n    # And the following too:\n    assert set(names).issuperset(DEFAULT_PLUGINS)\n    # --requirements should be empty because there are no installed non-plugins-dir plugins\n    result3 = runner.invoke(cli, [\"plugins\", \"--requirements\"])\n    assert result3.output == \"\"\n\n\ndef test_metadata_yaml():\n    yaml_file = io.StringIO(textwrap.dedent(\"\"\"\n    title: Hello from YAML\n    \"\"\"))\n    # Annoyingly we have to provide all default arguments here:\n    ds = serve.callback(\n        [],\n        metadata=yaml_file,\n        immutable=[],\n        host=\"127.0.0.1\",\n        port=8001,\n        uds=None,\n        reload=False,\n        cors=False,\n        sqlite_extensions=[],\n        inspect_file=None,\n        template_dir=None,\n        plugins_dir=None,\n        static=[],\n        memory=False,\n        config=[],\n        settings=[],\n        secret=None,\n        root=False,\n        default_deny=False,\n        token=None,\n        actor=None,\n        version_note=None,\n        get=None,\n        headers=False,\n        help_settings=False,\n        pdb=False,\n        crossdb=False,\n        nolock=False,\n        open_browser=False,\n        create=False,\n        ssl_keyfile=None,\n        ssl_certfile=None,\n        return_instance=True,\n        internal=None,\n    )\n    client = _TestClient(ds)\n    response = client.get(\"/.json\")\n    assert {\"title\": \"Hello from YAML\"} == response.json[\"metadata\"]\n\n\n@mock.patch(\"datasette.cli.run_module\")\ndef test_install(run_module):\n    runner = CliRunner()\n    runner.invoke(cli, [\"install\", \"datasette-mock-plugin\", \"datasette-mock-plugin2\"])\n    run_module.assert_called_once_with(\"pip\", run_name=\"__main__\")\n    assert sys.argv == [\n        \"pip\",\n        \"install\",\n        \"datasette-mock-plugin\",\n        \"datasette-mock-plugin2\",\n    ]\n\n\n@pytest.mark.parametrize(\"flag\", [\"-U\", \"--upgrade\"])\n@mock.patch(\"datasette.cli.run_module\")\ndef test_install_upgrade(run_module, flag):\n    runner = CliRunner()\n    runner.invoke(cli, [\"install\", flag, \"datasette\"])\n    run_module.assert_called_once_with(\"pip\", run_name=\"__main__\")\n    assert sys.argv == [\"pip\", \"install\", \"--upgrade\", \"datasette\"]\n\n\n@mock.patch(\"datasette.cli.run_module\")\ndef test_install_requirements(run_module, tmpdir):\n    path = tmpdir.join(\"requirements.txt\")\n    path.write(\"datasette-mock-plugin\\ndatasette-plugin-2\")\n    runner = CliRunner()\n    runner.invoke(cli, [\"install\", \"-r\", str(path)])\n    run_module.assert_called_once_with(\"pip\", run_name=\"__main__\")\n    assert sys.argv == [\"pip\", \"install\", \"-r\", str(path)]\n\n\ndef test_install_error_if_no_packages():\n    runner = CliRunner()\n    result = runner.invoke(cli, [\"install\"])\n    assert result.exit_code == 2\n    assert \"Error: Please specify at least one package to install\" in result.output\n\n\n@mock.patch(\"datasette.cli.run_module\")\ndef test_uninstall(run_module):\n    runner = CliRunner()\n    runner.invoke(cli, [\"uninstall\", \"datasette-mock-plugin\", \"-y\"])\n    run_module.assert_called_once_with(\"pip\", run_name=\"__main__\")\n    assert sys.argv == [\"pip\", \"uninstall\", \"datasette-mock-plugin\", \"-y\"]\n\n\ndef test_version():\n    runner = CliRunner()\n    result = runner.invoke(cli, [\"--version\"])\n    assert result.output == f\"cli, version {__version__}\\n\"\n\n\n@pytest.mark.parametrize(\"invalid_port\", [\"-1\", \"0.5\", \"dog\", \"65536\"])\ndef test_serve_invalid_ports(invalid_port):\n    runner = CliRunner()\n    result = runner.invoke(cli, [\"--port\", invalid_port])\n    assert result.exit_code == 2\n    assert \"Invalid value for '-p'\" in result.stderr\n\n\n@pytest.mark.parametrize(\n    \"args\",\n    (\n        [\"--setting\", \"default_page_size\", \"5\"],\n        [\"--setting\", \"settings.default_page_size\", \"5\"],\n        [\"-s\", \"settings.default_page_size\", \"5\"],\n    ),\n)\ndef test_setting(args):\n    runner = CliRunner()\n    result = runner.invoke(cli, [\"--get\", \"/-/settings.json\"] + args)\n    assert result.exit_code == 0, result.output\n    settings = json.loads(result.output)\n    assert settings[\"default_page_size\"] == 5\n\n\ndef test_setting_compatible_with_config(tmp_path):\n    # https://github.com/simonw/datasette/issues/2389\n    runner = CliRunner()\n    config_path = tmp_path / \"config.json\"\n    config_path.write_text(\n        '{\"settings\": {\"default_page_size\": 5, \"sql_time_limit_ms\": 50}}', \"utf-8\"\n    )\n    result = runner.invoke(\n        cli,\n        [\n            \"--get\",\n            \"/-/settings.json\",\n            \"--config\",\n            str(config_path),\n            \"--setting\",\n            \"default_page_size\",\n            \"10\",\n        ],\n    )\n    assert result.exit_code == 0, result.output\n    settings = json.loads(result.output)\n    assert settings[\"default_page_size\"] == 10\n    assert settings[\"sql_time_limit_ms\"] == 50\n\n\ndef test_plugin_s_overwrite():\n    runner = CliRunner()\n    plugins_dir = str(pathlib.Path(__file__).parent / \"plugins\")\n\n    result = runner.invoke(\n        cli,\n        [\n            \"--plugins-dir\",\n            plugins_dir,\n            \"--get\",\n            \"/_memory/-/query.json?sql=select+prepare_connection_args()\",\n        ],\n    )\n    assert result.exit_code == 0, result.output\n    assert (\n        json.loads(result.output).get(\"rows\")[0].get(\"prepare_connection_args()\")\n        == 'database=_memory, datasette.plugin_config(\"name-of-plugin\")=None'\n    )\n\n    result = runner.invoke(\n        cli,\n        [\n            \"--plugins-dir\",\n            plugins_dir,\n            \"--get\",\n            \"/_memory/-/query.json?sql=select+prepare_connection_args()\",\n            \"-s\",\n            \"plugins.name-of-plugin\",\n            \"OVERRIDE\",\n        ],\n    )\n    assert result.exit_code == 0, result.output\n    assert (\n        json.loads(result.output).get(\"rows\")[0].get(\"prepare_connection_args()\")\n        == 'database=_memory, datasette.plugin_config(\"name-of-plugin\")=OVERRIDE'\n    )\n\n\ndef test_startup_error_from_plugin_is_click_exception(tmp_path):\n    plugins_dir = tmp_path / \"plugins\"\n    plugins_dir.mkdir()\n    (plugins_dir / \"startup_error.py\").write_text(\n        \"from datasette import hookimpl\\n\"\n        \"from datasette.utils import StartupError\\n\"\n        \"\\n\"\n        \"@hookimpl\\n\"\n        \"def startup(datasette):\\n\"\n        '    raise StartupError(\"boom\")\\n',\n        \"utf-8\",\n    )\n    runner = CliRunner()\n    result = runner.invoke(\n        cli,\n        [\n            \"--plugins-dir\",\n            str(plugins_dir),\n            \"--get\",\n            \"/\",\n        ],\n    )\n    try:\n        assert result.exit_code == 1\n        assert \"Error: boom\" in result.output\n    finally:\n        # Cleanup: Unregister the plugin to avoid test isolation issues\n        to_unregister = [\n            p for p in pm.get_plugins() if p.__name__ == \"startup_error.py\"\n        ]\n        if to_unregister:\n            pm.unregister(to_unregister[0])\n\n\ndef test_setting_type_validation():\n    runner = CliRunner()\n    result = runner.invoke(cli, [\"--setting\", \"default_page_size\", \"dog\"])\n    assert result.exit_code == 2\n    assert '\"settings.default_page_size\" should be an integer' in result.output\n\n\ndef test_setting_boolean_validation_invalid():\n    \"\"\"Test that invalid boolean values are rejected\"\"\"\n    runner = CliRunner()\n    result = runner.invoke(\n        cli, [\"--setting\", \"default_allow_sql\", \"invalid\", \"--get\", \"/-/settings.json\"]\n    )\n    assert result.exit_code == 2\n    assert (\n        '\"settings.default_allow_sql\" should be on/off/true/false/1/0' in result.output\n    )\n\n\n@pytest.mark.parametrize(\"value\", (\"off\", \"false\", \"0\"))\ndef test_setting_boolean_validation_false_values(value):\n    \"\"\"Test that 'off', 'false', '0' work for boolean settings\"\"\"\n    runner = CliRunner()\n    result = runner.invoke(\n        cli,\n        [\n            \"--setting\",\n            \"default_allow_sql\",\n            value,\n            \"--get\",\n            \"/_memory/-/query.json?sql=select+1\",\n        ],\n    )\n    # Should be forbidden (setting is false)\n    assert result.exit_code == 1, result.output\n    assert \"Forbidden\" in result.output\n\n\n@pytest.mark.parametrize(\"value\", (\"on\", \"true\", \"1\"))\ndef test_setting_boolean_validation_true_values(value):\n    \"\"\"Test that 'on', 'true', '1' work for boolean settings\"\"\"\n    runner = CliRunner()\n    result = runner.invoke(\n        cli,\n        [\n            \"--setting\",\n            \"default_allow_sql\",\n            value,\n            \"--get\",\n            \"/_memory/-/query.json?sql=select+1&_shape=objects\",\n        ],\n    )\n    # Should succeed (setting is true)\n    assert result.exit_code == 0, result.output\n    assert json.loads(result.output)[\"rows\"][0] == {\"1\": 1}\n\n\n@pytest.mark.parametrize(\"default_allow_sql\", (True, False))\ndef test_setting_default_allow_sql(default_allow_sql):\n    runner = CliRunner()\n    result = runner.invoke(\n        cli,\n        [\n            \"--setting\",\n            \"default_allow_sql\",\n            \"on\" if default_allow_sql else \"off\",\n            \"--get\",\n            \"/_memory/-/query.json?sql=select+21&_shape=objects\",\n        ],\n    )\n    if default_allow_sql:\n        assert result.exit_code == 0, result.output\n        assert json.loads(result.output)[\"rows\"][0] == {\"21\": 21}\n    else:\n        assert result.exit_code == 1, result.output\n        # This isn't JSON at the moment, maybe it should be though\n        assert \"Forbidden\" in result.output\n\n\ndef test_sql_errors_logged_to_stderr():\n    runner = CliRunner()\n    result = runner.invoke(cli, [\"--get\", \"/_memory/-/query.json?sql=select+blah\"])\n    assert result.exit_code == 1\n    assert \"sql = 'select blah', params = {}: no such column: blah\\n\" in result.stderr\n\n\ndef test_serve_create(tmpdir):\n    runner = CliRunner()\n    db_path = tmpdir / \"does_not_exist_yet.db\"\n    assert not db_path.exists()\n    result = runner.invoke(\n        cli, [str(db_path), \"--create\", \"--get\", \"/-/databases.json\"]\n    )\n    assert result.exit_code == 0, result.output\n    databases = json.loads(result.output)\n    assert {\n        \"name\": \"does_not_exist_yet\",\n        \"is_mutable\": True,\n        \"is_memory\": False,\n        \"hash\": None,\n    }.items() <= databases[0].items()\n    assert db_path.exists()\n\n\n@pytest.mark.parametrize(\"argument\", (\"-c\", \"--config\"))\n@pytest.mark.parametrize(\"format_\", (\"json\", \"yaml\"))\ndef test_serve_config(tmpdir, argument, format_):\n    config_path = tmpdir / \"datasette.{}\".format(format_)\n    config_path.write_text(\n        (\n            \"settings:\\n  default_page_size: 5\\n\"\n            if format_ == \"yaml\"\n            else '{\"settings\": {\"default_page_size\": 5}}'\n        ),\n        \"utf-8\",\n    )\n    runner = CliRunner()\n    result = runner.invoke(\n        cli,\n        [\n            argument,\n            str(config_path),\n            \"--get\",\n            \"/-/settings.json\",\n        ],\n    )\n    assert result.exit_code == 0, result.output\n    assert json.loads(result.output)[\"default_page_size\"] == 5\n\n\ndef test_serve_duplicate_database_names(tmpdir):\n    \"'datasette db.db nested/db.db' should attach two databases, /db and /db_2\"\n    runner = CliRunner()\n    db_1_path = str(tmpdir / \"db.db\")\n    nested = tmpdir / \"nested\"\n    nested.mkdir()\n    db_2_path = str(tmpdir / \"nested\" / \"db.db\")\n    for path in (db_1_path, db_2_path):\n        sqlite3.connect(path).execute(\"vacuum\")\n    result = runner.invoke(cli, [db_1_path, db_2_path, \"--get\", \"/-/databases.json\"])\n    assert result.exit_code == 0, result.output\n    databases = json.loads(result.output)\n    assert {db[\"name\"] for db in databases} == {\"db\", \"db_2\"}\n\n\n@pytest.mark.parametrize(\n    \"filename\", [\"test-database (1).sqlite\", \"database (1).sqlite\"]\n)\ndef test_weird_database_names(tmpdir, filename):\n    # https://github.com/simonw/datasette/issues/1181\n    runner = CliRunner()\n    db_path = str(tmpdir / filename)\n    sqlite3.connect(db_path).execute(\"vacuum\")\n    result1 = runner.invoke(cli, [db_path, \"--get\", \"/\"])\n    assert result1.exit_code == 0, result1.output\n    filename_no_stem = filename.rsplit(\".\", 1)[0]\n    expected_link = '{}'.format(\n        tilde_encode(filename_no_stem), filename_no_stem\n    )\n    assert expected_link in result1.output\n    # Now try hitting that database page\n    result2 = runner.invoke(\n        cli, [db_path, \"--get\", \"/{}\".format(tilde_encode(filename_no_stem))]\n    )\n    assert result2.exit_code == 0, result2.output\n\n\ndef test_help_settings():\n    runner = CliRunner()\n    result = runner.invoke(cli, [\"--help-settings\"])\n    for setting in SETTINGS:\n        assert setting.name in result.output\n\n\ndef test_internal_db(tmpdir):\n    runner = CliRunner()\n    internal_path = tmpdir / \"internal.db\"\n    assert not internal_path.exists()\n    result = runner.invoke(\n        cli, [\"--memory\", \"--internal\", str(internal_path), \"--get\", \"/\"]\n    )\n    assert result.exit_code == 0\n    assert internal_path.exists()\n\n\ndef test_duplicate_database_files_error(tmpdir):\n    \"\"\"Test that passing the same database file multiple times raises an error\"\"\"\n    runner = CliRunner()\n    db_path = str(tmpdir / \"test.db\")\n    sqlite3.connect(db_path).execute(\"vacuum\")\n\n    # Test with exact duplicate\n    result = runner.invoke(cli, [\"serve\", db_path, db_path, \"--get\", \"/\"])\n    assert result.exit_code == 1\n    assert \"Duplicate database file\" in result.output\n    assert \"both refer to\" in result.output\n\n    # Test with different paths to same file (relative vs absolute)\n    result2 = runner.invoke(\n        cli, [\"serve\", db_path, str(pathlib.Path(db_path).resolve()), \"--get\", \"/\"]\n    )\n    assert result2.exit_code == 1\n    assert \"Duplicate database file\" in result2.output\n\n    # Test that a file in the config_dir can't also be passed explicitly\n    config_dir = tmpdir / \"config\"\n    config_dir.mkdir()\n    config_db_path = str(config_dir / \"data.db\")\n    sqlite3.connect(config_db_path).execute(\"vacuum\")\n\n    result3 = runner.invoke(\n        cli, [\"serve\", config_db_path, str(config_dir), \"--get\", \"/\"]\n    )\n    assert result3.exit_code == 1\n    assert \"Duplicate database file\" in result3.output\n    assert \"both refer to\" in result3.output\n\n    # Test that mixing a file NOT in the directory with a directory works fine\n    other_db_path = str(tmpdir / \"other.db\")\n    sqlite3.connect(other_db_path).execute(\"vacuum\")\n\n    result4 = runner.invoke(\n        cli, [\"serve\", other_db_path, str(config_dir), \"--get\", \"/-/databases.json\"]\n    )\n    assert result4.exit_code == 0\n    databases = json.loads(result4.output)\n    assert {db[\"name\"] for db in databases} == {\"other\", \"data\"}\n\n    # Test that multiple directories raise an error\n    config_dir2 = tmpdir / \"config2\"\n    config_dir2.mkdir()\n\n    result5 = runner.invoke(\n        cli, [\"serve\", str(config_dir), str(config_dir2), \"--get\", \"/\"]\n    )\n    assert result5.exit_code == 1\n    assert \"Cannot pass multiple directories\" in result5.output\n"
  },
  {
    "path": "tests/test_cli_serve_get.py",
    "content": "from datasette.cli import cli\nfrom datasette.plugins import pm\nfrom click.testing import CliRunner\nimport textwrap\nimport json\n\n\ndef test_serve_with_get(tmp_path_factory):\n    plugins_dir = tmp_path_factory.mktemp(\"plugins_for_serve_with_get\")\n    (plugins_dir / \"init_for_serve_with_get.py\").write_text(\n        textwrap.dedent(\n            \"\"\"\n        from datasette import hookimpl\n\n        @hookimpl\n        def startup(datasette):\n            with open(\"{}\", \"w\") as fp:\n                fp.write(\"hello\")\n    \"\"\".format(str(plugins_dir / \"hello.txt\")),\n        ),\n        \"utf-8\",\n    )\n    runner = CliRunner()\n    result = runner.invoke(\n        cli,\n        [\n            \"serve\",\n            \"--memory\",\n            \"--plugins-dir\",\n            str(plugins_dir),\n            \"--get\",\n            \"/_memory/-/query.json?sql=select+sqlite_version()\",\n        ],\n    )\n    assert result.exit_code == 0, result.output\n    data = json.loads(result.output)\n    # Should have a single row with a single column\n    assert len(data[\"rows\"]) == 1\n    assert list(data[\"rows\"][0].keys()) == [\"sqlite_version()\"]\n    assert set(data.keys()) == {\"rows\", \"ok\", \"truncated\"}\n\n    # The plugin should have created hello.txt\n    assert (plugins_dir / \"hello.txt\").read_text() == \"hello\"\n\n    # Annoyingly that new test plugin stays resident - we need\n    # to manually unregister it to avoid conflict with other tests\n    to_unregister = [\n        p for p in pm.get_plugins() if p.__name__ == \"init_for_serve_with_get.py\"\n    ][0]\n    pm.unregister(to_unregister)\n\n\ndef test_serve_with_get_headers():\n    runner = CliRunner()\n    result = runner.invoke(\n        cli,\n        [\n            \"serve\",\n            \"--memory\",\n            \"--get\",\n            \"/_memory/\",\n            \"--headers\",\n        ],\n    )\n    # exit_code is 1 because it wasn't a 200 response\n    assert result.exit_code == 1, result.output\n    lines = result.output.splitlines()\n    assert lines and lines[0] == \"HTTP/1.1 302\"\n    assert \"location: /_memory\" in lines\n    assert \"content-type: text/html; charset=utf-8\" in lines\n\n\ndef test_serve_with_get_and_token():\n    runner = CliRunner()\n    result1 = runner.invoke(\n        cli,\n        [\n            \"create-token\",\n            \"--secret\",\n            \"sekrit\",\n            \"root\",\n        ],\n    )\n    token = result1.output.strip()\n    result2 = runner.invoke(\n        cli,\n        [\n            \"serve\",\n            \"--secret\",\n            \"sekrit\",\n            \"--get\",\n            \"/-/actor.json\",\n            \"--token\",\n            token,\n        ],\n    )\n    assert 0 == result2.exit_code, result2.output\n    assert json.loads(result2.output) == {\"actor\": {\"id\": \"root\", \"token\": \"dstok\"}}\n\n\ndef test_serve_with_get_exit_code_for_error():\n    runner = CliRunner()\n    result = runner.invoke(\n        cli,\n        [\n            \"serve\",\n            \"--memory\",\n            \"--get\",\n            \"/this-is-404\",\n        ],\n        catch_exceptions=False,\n    )\n    assert result.exit_code == 1\n    assert \"404\" in result.output\n\n\ndef test_serve_get_actor():\n    runner = CliRunner()\n    result = runner.invoke(\n        cli,\n        [\n            \"serve\",\n            \"--memory\",\n            \"--get\",\n            \"/-/actor.json\",\n            \"--actor\",\n            '{\"id\": \"root\", \"extra\": \"x\"}',\n        ],\n        catch_exceptions=False,\n    )\n    assert result.exit_code == 0\n    assert json.loads(result.output) == {\n        \"actor\": {\n            \"id\": \"root\",\n            \"extra\": \"x\",\n        }\n    }\n"
  },
  {
    "path": "tests/test_cli_serve_server.py",
    "content": "import httpx\nimport pytest\nimport socket\n\n\n@pytest.mark.serial\ndef test_serve_localhost_http(ds_localhost_http_server):\n    response = httpx.get(\"http://localhost:8041/_memory.json\")\n    assert {\n        \"database\": \"_memory\",\n        \"path\": \"/_memory\",\n        \"tables\": [],\n    }.items() <= response.json().items()\n\n\n@pytest.mark.serial\n@pytest.mark.skipif(\n    not hasattr(socket, \"AF_UNIX\"), reason=\"Requires socket.AF_UNIX support\"\n)\ndef test_serve_unix_domain_socket(ds_unix_domain_socket_server):\n    _, uds = ds_unix_domain_socket_server\n    transport = httpx.HTTPTransport(uds=uds)\n    client = httpx.Client(transport=transport)\n    response = client.get(\"http://localhost/_memory.json\")\n    assert {\n        \"database\": \"_memory\",\n        \"path\": \"/_memory\",\n        \"tables\": [],\n    }.items() <= response.json().items()\n"
  },
  {
    "path": "tests/test_column_types.py",
    "content": "import json\nimport logging\n\nfrom bs4 import BeautifulSoup as Soup\nfrom datasette.app import Datasette\nfrom datasette.column_types import (\n    ColumnType,\n    SQLiteType,\n)\nfrom datasette.hookspecs import hookimpl\nfrom datasette.plugins import pm\nfrom datasette.utils import sqlite3\nfrom datasette.utils import StartupError\nimport markupsafe\nimport pytest\nimport time\n\n\n@pytest.fixture\ndef ds_ct(tmp_path_factory):\n    db_directory = tmp_path_factory.mktemp(\"dbs\")\n    db_path = str(db_directory / \"data.db\")\n    db = sqlite3.connect(str(db_path))\n    db.execute(\"vacuum\")\n    db.execute(\n        \"create table posts (id integer primary key, title text, body text, \"\n        \"author_email text, website text, metadata text)\"\n    )\n    db.execute(\n        \"insert into posts values (1, 'Hello', '# World', 'test@example.com', \"\n        \"'https://example.com', '{\\\"key\\\": \\\"value\\\"}')\"\n    )\n    db.commit()\n    ds = Datasette(\n        [db_path],\n        config={\n            \"databases\": {\n                \"data\": {\n                    \"tables\": {\n                        \"posts\": {\n                            \"column_types\": {\n                                \"body\": \"markdown\",\n                                \"author_email\": \"email\",\n                                \"website\": \"url\",\n                                \"metadata\": \"json\",\n                            }\n                        }\n                    }\n                }\n            }\n        },\n    )\n    ds.root_enabled = True\n    yield ds\n    db.close()\n    for database in ds.databases.values():\n        if not database.is_memory:\n            database.close()\n\n\n@pytest.fixture\ndef ds_ct_editor_permission(tmp_path_factory):\n    db_directory = tmp_path_factory.mktemp(\"dbs\")\n    db_path = str(db_directory / \"data.db\")\n    db = sqlite3.connect(str(db_path))\n    db.execute(\"vacuum\")\n    db.execute(\n        \"create table posts (id integer primary key, title text, body text, \"\n        \"author_email text, website text, metadata text)\"\n    )\n    db.execute(\n        \"insert into posts values (1, 'Hello', '# World', 'test@example.com', \"\n        \"'https://example.com', '{\\\"key\\\": \\\"value\\\"}')\"\n    )\n    db.commit()\n    ds = Datasette(\n        [db_path],\n        config={\n            \"databases\": {\n                \"data\": {\n                    \"tables\": {\n                        \"posts\": {\n                            \"permissions\": {\"set-column-type\": {\"id\": \"editor\"}},\n                            \"column_types\": {\n                                \"body\": \"markdown\",\n                                \"author_email\": \"email\",\n                                \"website\": \"url\",\n                                \"metadata\": \"json\",\n                            },\n                        }\n                    }\n                }\n            }\n        },\n    )\n    ds.root_enabled = True\n    yield ds\n    db.close()\n    for database in ds.databases.values():\n        if not database.is_memory:\n            database.close()\n\n\ndef write_token(ds, actor_id=\"root\", permissions=None):\n    to_sign = {\"a\": actor_id, \"token\": \"dstok\", \"t\": int(time.time())}\n    if permissions:\n        to_sign[\"_r\"] = {\"a\": permissions}\n    return \"dstok_{}\".format(ds.sign(to_sign, namespace=\"token\"))\n\n\ndef _headers(token):\n    return {\n        \"Authorization\": \"Bearer {}\".format(token),\n        \"Content-Type\": \"application/json\",\n    }\n\n\ndef _window_data_from_html(html, variable_name):\n    soup = Soup(html, \"html.parser\")\n    scripts = soup.find_all(\"script\")\n    matching_scripts = [\n        script for script in scripts if variable_name in (script.string or \"\")\n    ]\n    assert len(matching_scripts) == 1\n    script_text = matching_scripts[0].string.strip()\n    prefix = f\"window.{variable_name} = \"\n    assert script_text.startswith(prefix)\n    return json.loads(script_text[len(prefix) :].rstrip(\";\"))\n\n\n# --- Internal DB and config loading ---\n\n\n@pytest.mark.asyncio\nasync def test_column_types_table_created(ds_ct):\n    await ds_ct.invoke_startup()\n    internal = ds_ct.get_internal_database()\n    result = await internal.execute(\n        \"SELECT name FROM sqlite_master WHERE type='table' AND name='column_types'\"\n    )\n    assert len(result.rows) == 1\n\n\n@pytest.mark.asyncio\nasync def test_config_loaded_into_internal_db(ds_ct):\n    await ds_ct.invoke_startup()\n    ct_map = await ds_ct.get_column_types(\"data\", \"posts\")\n    # \"markdown\" is not a registered type, so it won't appear\n    assert \"body\" not in ct_map\n    assert ct_map[\"author_email\"].name == \"email\"\n    assert ct_map[\"author_email\"].config is None\n    assert ct_map[\"website\"].name == \"url\"\n    assert ct_map[\"metadata\"].name == \"json\"\n\n\n@pytest.mark.asyncio\nasync def test_config_with_type_and_config(tmp_path_factory):\n    class PointColumnType(ColumnType):\n        name = \"point\"\n        description = \"Geographic point\"\n\n    class _Plugin:\n        @hookimpl\n        def register_column_types(self, datasette):\n            return [PointColumnType]\n\n    plugin = _Plugin()\n    pm.register(plugin, name=\"test_point_ct\")\n    try:\n        db_directory = tmp_path_factory.mktemp(\"dbs\")\n        db_path = str(db_directory / \"data.db\")\n        db = sqlite3.connect(str(db_path))\n        db.execute(\"vacuum\")\n        db.execute(\"create table geo (id integer primary key, location text)\")\n        ds = Datasette(\n            [db_path],\n            config={\n                \"databases\": {\n                    \"data\": {\n                        \"tables\": {\n                            \"geo\": {\n                                \"column_types\": {\n                                    \"location\": {\n                                        \"type\": \"point\",\n                                        \"config\": {\"srid\": 4326},\n                                    }\n                                }\n                            }\n                        }\n                    }\n                }\n            },\n        )\n        await ds.invoke_startup()\n        ct = await ds.get_column_type(\"data\", \"geo\", \"location\")\n        assert ct.name == \"point\"\n        assert ct.config == {\"srid\": 4326}\n        db.close()\n        for database in ds.databases.values():\n            if not database.is_memory:\n                database.close()\n    finally:\n        pm.unregister(plugin, name=\"test_point_ct\")\n\n\n# --- Datasette API methods ---\n\n\n@pytest.mark.asyncio\nasync def test_get_column_type(ds_ct):\n    await ds_ct.invoke_startup()\n    ct = await ds_ct.get_column_type(\"data\", \"posts\", \"author_email\")\n    assert isinstance(ct, ColumnType)\n    assert ct.name == \"email\"\n    assert ct.config is None\n\n\n@pytest.mark.asyncio\nasync def test_get_column_type_missing(ds_ct):\n    await ds_ct.invoke_startup()\n    ct = await ds_ct.get_column_type(\"data\", \"posts\", \"title\")\n    assert ct is None\n\n\n@pytest.mark.asyncio\nasync def test_set_and_remove_column_type(ds_ct):\n    await ds_ct.invoke_startup()\n    await ds_ct.set_column_type(\"data\", \"posts\", \"title\", \"email\")\n    ct = await ds_ct.get_column_type(\"data\", \"posts\", \"title\")\n    assert ct.name == \"email\"\n    assert ct.config is None\n\n    await ds_ct.remove_column_type(\"data\", \"posts\", \"title\")\n    ct = await ds_ct.get_column_type(\"data\", \"posts\", \"title\")\n    assert ct is None\n\n\n@pytest.mark.asyncio\nasync def test_set_column_type_with_config(ds_ct):\n    await ds_ct.invoke_startup()\n    await ds_ct.set_column_type(\"data\", \"posts\", \"title\", \"url\", {\"max_length\": 200})\n    ct = await ds_ct.get_column_type(\"data\", \"posts\", \"title\")\n    assert ct.name == \"url\"\n    assert ct.config == {\"max_length\": 200}\n\n\n@pytest.mark.asyncio\nasync def test_set_column_type_api(ds_ct):\n    await ds_ct.invoke_startup()\n    token = write_token(ds_ct, permissions=[\"sct\"])\n    response = await ds_ct.client.post(\n        \"/data/posts/-/set-column-type\",\n        json={\"column\": \"title\", \"column_type\": {\"type\": \"email\"}},\n        headers=_headers(token),\n    )\n    assert response.status_code == 200\n    assert response.json() == {\n        \"ok\": True,\n        \"database\": \"data\",\n        \"table\": \"posts\",\n        \"column\": \"title\",\n        \"column_type\": {\"type\": \"email\", \"config\": None},\n    }\n    ct = await ds_ct.get_column_type(\"data\", \"posts\", \"title\")\n    assert ct.name == \"email\"\n    assert ct.config is None\n\n\n@pytest.mark.asyncio\nasync def test_set_column_type_api_with_config(ds_ct):\n    await ds_ct.invoke_startup()\n    token = write_token(ds_ct, permissions=[\"sct\"])\n    response = await ds_ct.client.post(\n        \"/data/posts/-/set-column-type\",\n        json={\n            \"column\": \"title\",\n            \"column_type\": {\"type\": \"url\", \"config\": {\"max_length\": 200}},\n        },\n        headers=_headers(token),\n    )\n    assert response.status_code == 200\n    assert response.json()[\"column_type\"] == {\n        \"type\": \"url\",\n        \"config\": {\"max_length\": 200},\n    }\n    ct = await ds_ct.get_column_type(\"data\", \"posts\", \"title\")\n    assert ct.name == \"url\"\n    assert ct.config == {\"max_length\": 200}\n\n\n@pytest.mark.asyncio\nasync def test_clear_column_type_api(ds_ct):\n    await ds_ct.invoke_startup()\n    await ds_ct.set_column_type(\"data\", \"posts\", \"title\", \"email\")\n    token = write_token(ds_ct, permissions=[\"sct\"])\n    response = await ds_ct.client.post(\n        \"/data/posts/-/set-column-type\",\n        json={\"column\": \"title\", \"column_type\": None},\n        headers=_headers(token),\n    )\n    assert response.status_code == 200\n    assert response.json() == {\n        \"ok\": True,\n        \"database\": \"data\",\n        \"table\": \"posts\",\n        \"column\": \"title\",\n        \"column_type\": None,\n    }\n    ct = await ds_ct.get_column_type(\"data\", \"posts\", \"title\")\n    assert ct is None\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"body,special_case,expected_status,expected_errors\",\n    (\n        (\n            {\"column\": \"title\", \"column_type\": {\"type\": \"email\"}},\n            \"no_permission\",\n            403,\n            [\"Permission denied\"],\n        ),\n        (\n            None,\n            \"invalid_json\",\n            400,\n            [\n                \"Invalid JSON: Expecting property name enclosed in double quotes: line 1 column 2 (char 1)\"\n            ],\n        ),\n        (\n            {\"column\": \"title\", \"column_type\": {\"type\": \"email\"}},\n            \"invalid_content_type\",\n            400,\n            [\"Invalid content-type, must be application/json\"],\n        ),\n        (\n            [],\n            None,\n            400,\n            [\"JSON must be a dictionary\"],\n        ),\n        (\n            {\"column_type\": {\"type\": \"email\"}},\n            None,\n            400,\n            ['\"column\" is required'],\n        ),\n        (\n            {\"column\": 1, \"column_type\": {\"type\": \"email\"}},\n            None,\n            400,\n            ['\"column\" must be a string'],\n        ),\n        (\n            {\"column\": \"not_a_column\", \"column_type\": {\"type\": \"email\"}},\n            None,\n            400,\n            [\"Column not found: not_a_column\"],\n        ),\n        (\n            {\"column\": \"title\", \"column_type\": \"email\"},\n            None,\n            400,\n            ['\"column_type\" must be an object or null'],\n        ),\n        (\n            {\"column\": \"title\", \"column_type\": {}},\n            None,\n            400,\n            ['\"column_type.type\" is required'],\n        ),\n        (\n            {\"column\": \"title\", \"column_type\": {\"type\": 1}},\n            None,\n            400,\n            ['\"column_type.type\" must be a string'],\n        ),\n        (\n            {\"column\": \"title\", \"column_type\": {\"type\": \"url\", \"config\": []}},\n            None,\n            400,\n            ['\"column_type.config\" must be a dictionary'],\n        ),\n        (\n            {\"column\": \"title\", \"column_type\": {\"type\": \"markdown\"}},\n            None,\n            400,\n            [\"Unknown column type: markdown\"],\n        ),\n        (\n            {\"column\": \"id\", \"column_type\": {\"type\": \"json\"}},\n            None,\n            400,\n            [\n                \"Column type 'json' is only applicable to SQLite types TEXT but data.posts.id has SQLite type INTEGER\"\n            ],\n        ),\n        (\n            {\n                \"column\": \"title\",\n                \"column_type\": {\"type\": \"email\"},\n                \"extra\": True,\n            },\n            None,\n            400,\n            ['Invalid parameter: \"extra\"'],\n        ),\n    ),\n)\nasync def test_set_column_type_api_errors(\n    ds_ct, body, special_case, expected_status, expected_errors\n):\n    await ds_ct.invoke_startup()\n    token = write_token(\n        ds_ct,\n        permissions=([\"sct\"] if special_case != \"no_permission\" else [\"vi\"]),\n    )\n    kwargs = {\n        \"headers\": {\n            \"Authorization\": f\"Bearer {token}\",\n            \"Content-Type\": (\n                \"text/plain\"\n                if special_case == \"invalid_content_type\"\n                else \"application/json\"\n            ),\n        }\n    }\n    if special_case == \"invalid_json\":\n        kwargs[\"content\"] = \"{bad json\"\n    else:\n        kwargs[\"json\"] = body\n    response = await ds_ct.client.post(\"/data/posts/-/set-column-type\", **kwargs)\n    assert response.status_code == expected_status\n    assert response.json() == {\"ok\": False, \"errors\": expected_errors}\n\n\n@pytest.mark.asyncio\nasync def test_set_column_type_api_works_for_immutable_database(tmp_path_factory):\n    db_directory = tmp_path_factory.mktemp(\"dbs\")\n    db_path = str(db_directory / \"immutable.db\")\n    db = sqlite3.connect(str(db_path))\n    db.execute(\"vacuum\")\n    db.execute(\"create table posts (id integer primary key, title text)\")\n    db.commit()\n    ds = Datasette([], immutables=[db_path])\n    ds.root_enabled = True\n    try:\n        await ds.invoke_startup()\n        token = write_token(ds, permissions=[\"sct\"])\n        response = await ds.client.post(\n            \"/immutable/posts/-/set-column-type\",\n            json={\"column\": \"title\", \"column_type\": {\"type\": \"email\"}},\n            headers=_headers(token),\n        )\n        assert response.status_code == 200\n        assert response.json()[\"column_type\"] == {\"type\": \"email\", \"config\": None}\n        ct = await ds.get_column_type(\"immutable\", \"posts\", \"title\")\n        assert ct.name == \"email\"\n    finally:\n        db.close()\n        for database in ds.databases.values():\n            if not database.is_memory:\n                database.close()\n\n\n@pytest.mark.asyncio\nasync def test_set_column_type_rejects_incompatible_sqlite_type(ds_ct):\n    await ds_ct.invoke_startup()\n    with pytest.raises(ValueError, match=\"only applicable to SQLite types TEXT\"):\n        await ds_ct.set_column_type(\"data\", \"posts\", \"id\", \"json\")\n\n\n@pytest.mark.asyncio\nasync def test_set_column_type_allows_varchar_for_text_only_type(tmp_path_factory):\n    db_directory = tmp_path_factory.mktemp(\"dbs\")\n    db_path = str(db_directory / \"data.db\")\n    db = sqlite3.connect(str(db_path))\n    db.execute(\"vacuum\")\n    db.execute(\"create table links (id integer primary key, url varchar(255))\")\n    db.commit()\n    ds = Datasette([db_path])\n    await ds.invoke_startup()\n    await ds.set_column_type(\"data\", \"links\", \"url\", \"url\")\n    ct = await ds.get_column_type(\"data\", \"links\", \"url\")\n    assert ct.name == \"url\"\n    db.close()\n    for database in ds.databases.values():\n        if not database.is_memory:\n            database.close()\n\n\n# --- Plugin registration ---\n\n\n@pytest.mark.asyncio\nasync def test_builtin_column_types_registered(ds_ct):\n    \"\"\"register_column_types returns classes; _column_types stores them by name.\"\"\"\n    await ds_ct.invoke_startup()\n    assert \"url\" in ds_ct._column_types\n    assert \"email\" in ds_ct._column_types\n    assert \"json\" in ds_ct._column_types\n    assert \"nonexistent\" not in ds_ct._column_types\n\n\n@pytest.mark.asyncio\nasync def test_column_type_class_attributes(ds_ct):\n    await ds_ct.invoke_startup()\n    url_cls = ds_ct._column_types[\"url\"]\n    assert url_cls.name == \"url\"\n    assert url_cls.description == \"URL\"\n    assert url_cls.sqlite_types == (SQLiteType.TEXT,)\n    email_cls = ds_ct._column_types[\"email\"]\n    assert email_cls.name == \"email\"\n    assert email_cls.description == \"Email address\"\n    assert email_cls.sqlite_types == (SQLiteType.TEXT,)\n    json_cls = ds_ct._column_types[\"json\"]\n    assert json_cls.sqlite_types == (SQLiteType.TEXT,)\n\n\ndef test_sqlite_type_from_declared_type():\n    assert SQLiteType.from_declared_type(\"text\") == SQLiteType.TEXT\n    assert SQLiteType.from_declared_type(\"varchar(255)\") == SQLiteType.TEXT\n    assert SQLiteType.from_declared_type(\"integer\") == SQLiteType.INTEGER\n    assert SQLiteType.from_declared_type(\"float\") == SQLiteType.REAL\n    assert SQLiteType.from_declared_type(\"blob\") == SQLiteType.BLOB\n    assert SQLiteType.from_declared_type(\"\") == SQLiteType.NULL\n    assert SQLiteType.from_declared_type(\"numeric\") is None\n\n\n# --- JSON API ---\n\n\n@pytest.mark.asyncio\nasync def test_column_types_extra(ds_ct):\n    await ds_ct.invoke_startup()\n    response = await ds_ct.client.get(\"/data/posts.json?_extra=column_types\")\n    assert response.status_code == 200\n    data = response.json()\n    assert \"column_types\" in data\n    assert data[\"column_types\"][\"author_email\"] == {\"type\": \"email\", \"config\": None}\n    assert data[\"column_types\"][\"website\"] == {\"type\": \"url\", \"config\": None}\n    assert data[\"column_types\"][\"metadata\"] == {\"type\": \"json\", \"config\": None}\n    # \"markdown\" is not a registered type, so body should not appear\n    assert \"body\" not in data[\"column_types\"]\n    # title has no column type, should not appear\n    assert \"title\" not in data[\"column_types\"]\n\n\n@pytest.mark.asyncio\nasync def test_display_columns_include_column_type(ds_ct):\n    await ds_ct.invoke_startup()\n    response = await ds_ct.client.get(\"/data/posts.json?_extra=display_columns\")\n    assert response.status_code == 200\n    data = response.json()\n    cols = {c[\"name\"]: c for c in data[\"display_columns\"]}\n    assert cols[\"author_email\"][\"column_type\"] == \"email\"\n    assert cols[\"author_email\"][\"column_type_config\"] is None\n    assert cols[\"website\"][\"column_type\"] == \"url\"\n    assert cols[\"title\"][\"column_type\"] is None\n\n\n# --- Rendering ---\n\n\n@pytest.mark.asyncio\nasync def test_url_render_cell(ds_ct):\n    await ds_ct.invoke_startup()\n    response = await ds_ct.client.get(\"/data/posts.json?_extra=render_cell\")\n    assert response.status_code == 200\n    data = response.json()\n    rendered = data[\"render_cell\"][0]\n    assert \"href\" in rendered[\"website\"]\n    assert \"https://example.com\" in rendered[\"website\"]\n\n\n@pytest.mark.asyncio\nasync def test_email_render_cell(ds_ct):\n    await ds_ct.invoke_startup()\n    response = await ds_ct.client.get(\"/data/posts.json?_extra=render_cell\")\n    assert response.status_code == 200\n    data = response.json()\n    rendered = data[\"render_cell\"][0]\n    assert \"mailto:\" in rendered[\"author_email\"]\n    assert \"test@example.com\" in rendered[\"author_email\"]\n\n\n@pytest.mark.asyncio\nasync def test_json_render_cell(ds_ct):\n    await ds_ct.invoke_startup()\n    response = await ds_ct.client.get(\"/data/posts.json?_extra=render_cell\")\n    assert response.status_code == 200\n    data = response.json()\n    rendered = data[\"render_cell\"][0]\n    assert \"
    \" in rendered[\"metadata\"]\n\n\n# --- Validation ---\n\n\n@pytest.mark.asyncio\nasync def test_email_validation_on_insert(ds_ct):\n    await ds_ct.invoke_startup()\n    token = write_token(ds_ct)\n    response = await ds_ct.client.post(\n        \"/data/posts/-/insert\",\n        json={\"row\": {\"title\": \"Test\", \"author_email\": \"not-an-email\"}},\n        headers=_headers(token),\n    )\n    assert response.status_code == 400\n    assert \"author_email\" in response.json()[\"errors\"][0]\n\n\n@pytest.mark.asyncio\nasync def test_email_validation_passes_valid(ds_ct):\n    await ds_ct.invoke_startup()\n    token = write_token(ds_ct)\n    response = await ds_ct.client.post(\n        \"/data/posts/-/insert\",\n        json={\"row\": {\"title\": \"Test\", \"author_email\": \"valid@example.com\"}},\n        headers=_headers(token),\n    )\n    assert response.status_code == 201\n\n\n@pytest.mark.asyncio\nasync def test_url_validation_on_insert(ds_ct):\n    await ds_ct.invoke_startup()\n    token = write_token(ds_ct)\n    response = await ds_ct.client.post(\n        \"/data/posts/-/insert\",\n        json={\"row\": {\"title\": \"Test\", \"website\": \"not-a-url\"}},\n        headers=_headers(token),\n    )\n    assert response.status_code == 400\n    assert \"website\" in response.json()[\"errors\"][0]\n\n\n@pytest.mark.asyncio\nasync def test_json_validation_on_insert(ds_ct):\n    await ds_ct.invoke_startup()\n    token = write_token(ds_ct)\n    response = await ds_ct.client.post(\n        \"/data/posts/-/insert\",\n        json={\"row\": {\"title\": \"Test\", \"metadata\": \"not-json{\"}},\n        headers=_headers(token),\n    )\n    assert response.status_code == 400\n    assert \"metadata\" in response.json()[\"errors\"][0]\n\n\n@pytest.mark.asyncio\nasync def test_validation_on_update(ds_ct):\n    await ds_ct.invoke_startup()\n    token = write_token(ds_ct)\n    response = await ds_ct.client.post(\n        \"/data/posts/1/-/update\",\n        json={\"update\": {\"author_email\": \"invalid\"}},\n        headers=_headers(token),\n    )\n    assert response.status_code == 400\n    assert \"author_email\" in response.json()[\"errors\"][0]\n\n\n@pytest.mark.asyncio\nasync def test_validation_allows_null(ds_ct):\n    await ds_ct.invoke_startup()\n    token = write_token(ds_ct)\n    response = await ds_ct.client.post(\n        \"/data/posts/-/insert\",\n        json={\"row\": {\"title\": \"Test\", \"author_email\": None}},\n        headers=_headers(token),\n    )\n    assert response.status_code == 201\n\n\n@pytest.mark.asyncio\nasync def test_validation_allows_empty_string(ds_ct):\n    await ds_ct.invoke_startup()\n    token = write_token(ds_ct)\n    response = await ds_ct.client.post(\n        \"/data/posts/-/insert\",\n        json={\"row\": {\"title\": \"Test\", \"author_email\": \"\"}},\n        headers=_headers(token),\n    )\n    assert response.status_code == 201\n\n\n# --- ColumnType base class ---\n\n\n@pytest.mark.asyncio\nasync def test_column_type_base_defaults():\n    class TestType(ColumnType):\n        name = \"test\"\n        description = \"Test type\"\n\n    ct = TestType()\n    assert ct.config is None\n    assert await ct.render_cell(\"val\", \"col\", \"tbl\", \"db\", None, None) is None\n    assert await ct.validate(\"val\", None) is None\n    assert await ct.transform_value(\"val\", None) == \"val\"\n\n\n# --- render_cell extra with column types ---\n\n\n@pytest.mark.asyncio\nasync def test_render_cell_extra_with_column_types(ds_ct):\n    await ds_ct.invoke_startup()\n    response = await ds_ct.client.get(\"/data/posts.json?_extra=render_cell\")\n    assert response.status_code == 200\n    data = response.json()\n    rendered = data[\"render_cell\"][0]\n    assert \"mailto:\" in rendered[\"author_email\"]\n    assert \"href\" in rendered[\"website\"]\n\n\n# --- Duplicate column type name ---\n\n\n@pytest.mark.asyncio\nasync def test_duplicate_column_type_name_raises_error():\n    class DuplicateUrlType(ColumnType):\n        name = \"url\"\n        description = \"Duplicate URL\"\n\n        async def render_cell(self, value, column, table, database, datasette, request):\n            return None\n\n    class _Plugin:\n        @hookimpl\n        def register_column_types(self, datasette):\n            return [DuplicateUrlType]\n\n    plugin = _Plugin()\n    pm.register(plugin, name=\"test_duplicate_ct\")\n    try:\n        ds = Datasette()\n        with pytest.raises(StartupError, match=\"Duplicate column type name: url\"):\n            await ds.invoke_startup()\n    finally:\n        pm.unregister(plugin, name=\"test_duplicate_ct\")\n\n\n# --- Row endpoint ---\n\n\n@pytest.mark.asyncio\nasync def test_row_endpoint_render_cell_with_column_types(ds_ct):\n    await ds_ct.invoke_startup()\n    response = await ds_ct.client.get(\"/data/posts/1.json?_extra=render_cell\")\n    assert response.status_code == 200\n    data = response.json()\n    rendered = data[\"render_cell\"][0]\n    assert \"mailto:\" in rendered[\"author_email\"]\n    assert \"href\" in rendered[\"website\"]\n\n\n# --- transform_value in JSON output ---\n\n\n@pytest.mark.asyncio\nasync def test_transform_value_in_json_output(tmp_path_factory):\n    \"\"\"A column type with transform_value should modify rows in JSON API.\"\"\"\n\n    class UpperColumnType(ColumnType):\n        name = \"upper\"\n        description = \"Uppercase\"\n\n        async def transform_value(self, value, datasette):\n            if isinstance(value, str):\n                return value.upper()\n            return value\n\n    class _Plugin:\n        @hookimpl\n        def register_column_types(self, datasette):\n            return [UpperColumnType]\n\n    plugin = _Plugin()\n    pm.register(plugin, name=\"test_transform_ct\")\n    try:\n        db_directory = tmp_path_factory.mktemp(\"dbs\")\n        db_path = str(db_directory / \"data.db\")\n        db = sqlite3.connect(str(db_path))\n        db.execute(\"vacuum\")\n        db.execute(\"create table t (id integer primary key, name text)\")\n        db.execute(\"insert into t values (1, 'hello')\")\n        db.commit()\n        ds = Datasette(\n            [db_path],\n            config={\n                \"databases\": {\n                    \"data\": {\"tables\": {\"t\": {\"column_types\": {\"name\": \"upper\"}}}}\n                }\n            },\n        )\n        await ds.invoke_startup()\n        response = await ds.client.get(\"/data/t.json\")\n        assert response.status_code == 200\n        data = response.json()\n        assert data[\"rows\"][0][\"name\"] == \"HELLO\"\n        db.close()\n        for database in ds.databases.values():\n            if not database.is_memory:\n                database.close()\n    finally:\n        pm.unregister(plugin, name=\"test_transform_ct\")\n\n\n# --- Column type priority over plugins ---\n\n\n@pytest.mark.asyncio\nasync def test_column_type_render_cell_has_priority_over_plugins(tmp_path_factory):\n    \"\"\"Column type render_cell should take priority over render_cell plugin hook.\"\"\"\n\n    class PriorityColumnType(ColumnType):\n        name = \"priority_test\"\n        description = \"Priority test\"\n\n        async def render_cell(self, value, column, table, database, datasette, request):\n            if value is not None:\n                return markupsafe.Markup(\n                    f\"COLUMN_TYPE:{markupsafe.escape(value)}\"\n                )\n            return None\n\n    class _ColumnTypePlugin:\n        @hookimpl\n        def register_column_types(self, datasette):\n            return [PriorityColumnType]\n\n    class _RenderCellPlugin:\n        @hookimpl\n        def render_cell(\n            self,\n            row,\n            value,\n            column,\n            table,\n            pks,\n            database,\n            datasette,\n            request,\n            column_type,\n        ):\n            if column == \"name\":\n                return markupsafe.Markup(f\"PLUGIN:{markupsafe.escape(value)}\")\n\n    ct_plugin = _ColumnTypePlugin()\n    rc_plugin = _RenderCellPlugin()\n    pm.register(ct_plugin, name=\"test_priority_ct\")\n    pm.register(rc_plugin, name=\"test_priority_render\")\n    try:\n        db_directory = tmp_path_factory.mktemp(\"dbs\")\n        db_path = str(db_directory / \"data.db\")\n        db = sqlite3.connect(str(db_path))\n        db.execute(\"vacuum\")\n        db.execute(\"create table t (id integer primary key, name text)\")\n        db.execute(\"insert into t values (1, 'hello')\")\n        db.commit()\n        ds = Datasette(\n            [db_path],\n            config={\n                \"databases\": {\n                    \"data\": {\n                        \"tables\": {\"t\": {\"column_types\": {\"name\": \"priority_test\"}}}\n                    }\n                }\n            },\n        )\n        await ds.invoke_startup()\n        response = await ds.client.get(\"/data/t.json?_extra=render_cell\")\n        assert response.status_code == 200\n        data = response.json()\n        rendered = data[\"render_cell\"][0]\n        # Column type should win over the plugin\n        assert \"COLUMN_TYPE:\" in rendered[\"name\"]\n        assert \"PLUGIN:\" not in rendered[\"name\"]\n        db.close()\n        for database in ds.databases.values():\n            if not database.is_memory:\n                database.close()\n    finally:\n        pm.unregister(ct_plugin, name=\"test_priority_ct\")\n        pm.unregister(rc_plugin, name=\"test_priority_render\")\n\n\n# --- Row detail page rendering ---\n\n\n@pytest.mark.asyncio\nasync def test_row_detail_page_html_rendering(ds_ct):\n    \"\"\"Row detail HTML page should use column type rendering.\"\"\"\n    await ds_ct.invoke_startup()\n    response = await ds_ct.client.get(\"/data/posts/1\")\n    assert response.status_code == 200\n    html = response.text\n    # The email column should be rendered with mailto: link\n    assert \"mailto:test@example.com\" in html\n    # The url column should be rendered with href\n    assert 'href=\"https://example.com\"' in html\n\n\n# --- HTML table page rendering ---\n\n\n@pytest.mark.asyncio\nasync def test_html_table_page_rendering(ds_ct):\n    \"\"\"HTML table page should use column type rendering.\"\"\"\n    await ds_ct.invoke_startup()\n    response = await ds_ct.client.get(\"/data/posts\")\n    assert response.status_code == 200\n    html = response.text\n    assert \"mailto:test@example.com\" in html\n    assert 'href=\"https://example.com\"' in html\n\n\n@pytest.mark.asyncio\nasync def test_set_column_type_ui_data_hidden_without_permission(ds_ct):\n    await ds_ct.invoke_startup()\n    response = await ds_ct.client.get(\"/data/posts\")\n    assert response.status_code == 200\n    assert \"window._setColumnTypeData\" not in response.text\n\n\n@pytest.mark.asyncio\nasync def test_set_column_type_ui_data_includes_applicable_types(\n    ds_ct_editor_permission,\n):\n    await ds_ct_editor_permission.invoke_startup()\n    response = await ds_ct_editor_permission.client.get(\n        \"/data/posts\",\n        cookies={\n            \"ds_actor\": ds_ct_editor_permission.client.actor_cookie({\"id\": \"editor\"})\n        },\n    )\n    assert response.status_code == 200\n    data = _window_data_from_html(response.text, \"_setColumnTypeData\")\n    assert data[\"path\"] == \"/data/posts/-/set-column-type\"\n    assert data[\"columns\"][\"id\"] == {\n        \"current\": None,\n        \"options\": [],\n    }\n    assert data[\"columns\"][\"title\"] == {\n        \"current\": None,\n        \"options\": [\n            {\"name\": \"email\", \"description\": \"Email address\"},\n            {\"name\": \"json\", \"description\": \"JSON data\"},\n            {\"name\": \"url\", \"description\": \"URL\"},\n        ],\n    }\n    assert data[\"columns\"][\"author_email\"] == {\n        \"current\": {\"type\": \"email\", \"config\": None},\n        \"options\": [\n            {\"name\": \"email\", \"description\": \"Email address\"},\n            {\"name\": \"json\", \"description\": \"JSON data\"},\n            {\"name\": \"url\", \"description\": \"URL\"},\n        ],\n    }\n\n\n# --- Validation on upsert ---\n\n\n@pytest.mark.asyncio\nasync def test_validation_on_upsert(ds_ct):\n    await ds_ct.invoke_startup()\n    token = write_token(ds_ct)\n    response = await ds_ct.client.post(\n        \"/data/posts/-/upsert\",\n        json={\n            \"rows\": [{\"id\": 1, \"title\": \"Updated\", \"author_email\": \"invalid\"}],\n        },\n        headers=_headers(token),\n    )\n    assert response.status_code == 400\n    assert \"author_email\" in response.json()[\"errors\"][0]\n\n\n@pytest.mark.asyncio\nasync def test_validation_on_upsert_passes_valid(ds_ct):\n    await ds_ct.invoke_startup()\n    token = write_token(ds_ct)\n    response = await ds_ct.client.post(\n        \"/data/posts/-/upsert\",\n        json={\n            \"rows\": [{\"id\": 1, \"title\": \"Updated\", \"author_email\": \"valid@test.com\"}],\n        },\n        headers=_headers(token),\n    )\n    assert response.status_code == 200\n\n\n# --- Unknown type warning logged ---\n\n\n@pytest.mark.asyncio\nasync def test_unknown_type_warning_logged(tmp_path_factory, caplog):\n    db_directory = tmp_path_factory.mktemp(\"dbs\")\n    db_path = str(db_directory / \"data.db\")\n    db = sqlite3.connect(str(db_path))\n    db.execute(\"vacuum\")\n    db.execute(\"create table t (id integer primary key, col text)\")\n    db.commit()\n    ds = Datasette(\n        [db_path],\n        config={\n            \"databases\": {\n                \"data\": {\"tables\": {\"t\": {\"column_types\": {\"col\": \"nonexistent_type\"}}}}\n            }\n        },\n    )\n    with caplog.at_level(logging.WARNING):\n        await ds.invoke_startup()\n    assert \"unknown type\" in caplog.text.lower()\n    assert \"nonexistent_type\" in caplog.text\n    db.close()\n    for database in ds.databases.values():\n        if not database.is_memory:\n            database.close()\n\n\n@pytest.mark.asyncio\nasync def test_incompatible_sqlite_type_warning_logged(tmp_path_factory, caplog):\n    db_directory = tmp_path_factory.mktemp(\"dbs\")\n    db_path = str(db_directory / \"data.db\")\n    db = sqlite3.connect(str(db_path))\n    db.execute(\"vacuum\")\n    db.execute(\"create table t (id integer primary key, col integer)\")\n    db.commit()\n    ds = Datasette(\n        [db_path],\n        config={\n            \"databases\": {\"data\": {\"tables\": {\"t\": {\"column_types\": {\"col\": \"json\"}}}}}\n        },\n    )\n    with caplog.at_level(logging.WARNING):\n        await ds.invoke_startup()\n    assert \"only applicable to sqlite types text\" in caplog.text.lower()\n    assert await ds.get_column_type(\"data\", \"t\", \"col\") is None\n    db.close()\n    for database in ds.databases.values():\n        if not database.is_memory:\n            database.close()\n\n\n# --- Config overwrites on restart ---\n\n\n@pytest.mark.asyncio\nasync def test_config_overwrites_on_restart(tmp_path_factory):\n    \"\"\"Config values should overwrite any existing column types in internal DB on startup.\"\"\"\n    db_directory = tmp_path_factory.mktemp(\"dbs\")\n    db_path = str(db_directory / \"data.db\")\n    db = sqlite3.connect(str(db_path))\n    db.execute(\"vacuum\")\n    db.execute(\"create table t (id integer primary key, col text)\")\n    db.commit()\n    ds = Datasette(\n        [db_path],\n        config={\n            \"databases\": {\"data\": {\"tables\": {\"t\": {\"column_types\": {\"col\": \"email\"}}}}}\n        },\n    )\n    await ds.invoke_startup()\n    ct = await ds.get_column_type(\"data\", \"t\", \"col\")\n    assert ct.name == \"email\"\n\n    # Manually change the column type in the internal DB\n    await ds.set_column_type(\"data\", \"t\", \"col\", \"url\")\n    ct = await ds.get_column_type(\"data\", \"t\", \"col\")\n    assert ct.name == \"url\"\n\n    # Re-apply config (simulating what happens on restart)\n    await ds._apply_column_types_config()\n    ct = await ds.get_column_type(\"data\", \"t\", \"col\")\n    assert ct.name == \"email\"  # Config wins\n\n    db.close()\n    for database in ds.databases.values():\n        if not database.is_memory:\n            database.close()\n\n\n# --- No column_types in config ---\n\n\n@pytest.mark.asyncio\nasync def test_no_column_types_in_config(tmp_path_factory):\n    \"\"\"Datasette should work fine without any column_types configuration.\"\"\"\n    db_directory = tmp_path_factory.mktemp(\"dbs\")\n    db_path = str(db_directory / \"data.db\")\n    db = sqlite3.connect(str(db_path))\n    db.execute(\"vacuum\")\n    db.execute(\"create table t (id integer primary key, col text)\")\n    db.execute(\"insert into t values (1, 'hello')\")\n    db.commit()\n    ds = Datasette([db_path])\n    await ds.invoke_startup()\n\n    # No column types assigned\n    ct_map = await ds.get_column_types(\"data\", \"t\")\n    assert ct_map == {}\n\n    # JSON endpoint should work without column_types extra\n    response = await ds.client.get(\"/data/t.json\")\n    assert response.status_code == 200\n    assert response.json()[\"rows\"][0][\"col\"] == \"hello\"\n\n    # column_types extra should return empty\n    response = await ds.client.get(\"/data/t.json?_extra=column_types\")\n    assert response.status_code == 200\n    assert response.json()[\"column_types\"] == {}\n\n    db.close()\n    for database in ds.databases.values():\n        if not database.is_memory:\n            database.close()\n"
  },
  {
    "path": "tests/test_config_dir.py",
    "content": "import json\nimport pathlib\nimport pytest\n\nfrom datasette.app import Datasette\nfrom datasette.utils.sqlite import sqlite3\nfrom datasette.utils import StartupError\nfrom .fixtures import TestClient as _TestClient\n\nPLUGIN = \"\"\"\nfrom datasette import hookimpl\n\n@hookimpl\ndef extra_template_vars():\n    return {\n        \"from_plugin\": \"hooray\"\n    }\n\"\"\"\nMETADATA = {\"title\": \"This is from metadata\"}\nCONFIG = {\n    \"settings\": {\n        \"default_cache_ttl\": 60,\n    }\n}\nCSS = \"\"\"\nbody { margin-top: 3em}\n\"\"\"\n\n\n@pytest.fixture(scope=\"session\")\ndef config_dir(tmp_path_factory):\n    config_dir = tmp_path_factory.mktemp(\"config-dir\")\n    plugins_dir = config_dir / \"plugins\"\n    plugins_dir.mkdir()\n    (plugins_dir / \"hooray.py\").write_text(PLUGIN, \"utf-8\")\n    (plugins_dir / \"non_py_file.txt\").write_text(PLUGIN, \"utf-8\")\n    (plugins_dir / \".mypy_cache\").mkdir()\n\n    templates_dir = config_dir / \"templates\"\n    templates_dir.mkdir()\n    (templates_dir / \"row.html\").write_text(\n        \"Show row here. Plugin says {{ from_plugin }}\", \"utf-8\"\n    )\n\n    static_dir = config_dir / \"static\"\n    static_dir.mkdir()\n    (static_dir / \"hello.css\").write_text(CSS, \"utf-8\")\n\n    (config_dir / \"metadata.json\").write_text(json.dumps(METADATA), \"utf-8\")\n    (config_dir / \"datasette.json\").write_text(json.dumps(CONFIG), \"utf-8\")\n\n    for dbname in (\"demo.db\", \"immutable.db\", \"j.sqlite3\", \"k.sqlite\"):\n        db = sqlite3.connect(str(config_dir / dbname))\n        db.executescript(\"\"\"\n        CREATE TABLE cities (\n            id integer primary key,\n            name text\n        );\n        INSERT INTO cities (id, name) VALUES\n            (1, 'San Francisco')\n        ;\n        \"\"\")\n\n    # Mark \"immutable.db\" as immutable\n    (config_dir / \"inspect-data.json\").write_text(\n        json.dumps(\n            {\n                \"immutable\": {\n                    \"hash\": \"hash\",\n                    \"size\": 8192,\n                    \"file\": \"immutable.db\",\n                    \"tables\": {\"cities\": {\"count\": 1}},\n                }\n            }\n        ),\n        \"utf-8\",\n    )\n    return config_dir\n\n\ndef test_invalid_settings(config_dir):\n    previous = (config_dir / \"datasette.json\").read_text(\"utf-8\")\n    (config_dir / \"datasette.json\").write_text(\n        json.dumps({\"settings\": {\"invalid\": \"invalid-setting\"}}), \"utf-8\"\n    )\n    try:\n        with pytest.raises(StartupError) as ex:\n            Datasette([], config_dir=config_dir)\n        assert ex.value.args[0] == \"Invalid setting 'invalid' in config file\"\n    finally:\n        (config_dir / \"datasette.json\").write_text(previous, \"utf-8\")\n\n\n@pytest.fixture(scope=\"session\")\ndef config_dir_client(config_dir):\n    ds = Datasette([], config_dir=config_dir)\n    yield _TestClient(ds)\n\n\ndef test_settings(config_dir_client):\n    response = config_dir_client.get(\"/-/settings.json\")\n    assert 200 == response.status\n    assert 60 == response.json[\"default_cache_ttl\"]\n\n\ndef test_plugins(config_dir_client):\n    response = config_dir_client.get(\"/-/plugins.json\")\n    assert 200 == response.status\n    assert \"hooray.py\" in {p[\"name\"] for p in response.json}\n    assert \"non_py_file.txt\" not in {p[\"name\"] for p in response.json}\n    assert \"mypy_cache\" not in {p[\"name\"] for p in response.json}\n\n\ndef test_templates_and_plugin(config_dir_client):\n    response = config_dir_client.get(\"/demo/cities/1\")\n    assert 200 == response.status\n    assert \"Show row here. Plugin says hooray\" == response.text\n\n\ndef test_static(config_dir_client):\n    response = config_dir_client.get(\"/static/hello.css\")\n    assert 200 == response.status\n    assert CSS == response.text\n    assert \"text/css\" == response.headers[\"content-type\"]\n\n\ndef test_static_directory_browsing_not_allowed(config_dir_client):\n    response = config_dir_client.get(\"/static/\")\n    assert 403 == response.status\n    assert \"403: Directory listing is not allowed\" == response.text\n\n\ndef test_databases(config_dir_client):\n    response = config_dir_client.get(\"/-/databases.json\")\n    assert 200 == response.status\n    databases = response.json\n    assert 4 == len(databases)\n    databases.sort(key=lambda d: d[\"name\"])\n    for db, expected_name in zip(databases, (\"demo\", \"immutable\", \"j\", \"k\")):\n        assert expected_name == db[\"name\"]\n        assert db[\"is_mutable\"] == (expected_name != \"immutable\")\n\n\ndef test_store_config_dir(config_dir_client):\n    ds = config_dir_client.ds\n\n    assert hasattr(ds, \"config_dir\")\n    assert ds.config_dir is not None\n    assert isinstance(ds.config_dir, pathlib.Path)\n"
  },
  {
    "path": "tests/test_config_permission_rules.py",
    "content": "import pytest\n\nfrom datasette.app import Datasette\nfrom datasette.database import Database\nfrom datasette.resources import DatabaseResource, TableResource\n\n\nasync def setup_datasette(config=None, databases=None):\n    ds = Datasette(memory=True, config=config)\n    for name in databases or []:\n        ds.add_database(Database(ds, memory_name=f\"{name}_memory\"), name=name)\n    await ds.invoke_startup()\n    await ds.refresh_schemas()\n    return ds\n\n\n@pytest.mark.asyncio\nasync def test_root_permissions_allow():\n    config = {\"permissions\": {\"execute-sql\": {\"id\": \"alice\"}}}\n    ds = await setup_datasette(config=config, databases=[\"content\"])\n\n    assert await ds.allowed(\n        action=\"execute-sql\",\n        resource=DatabaseResource(database=\"content\"),\n        actor={\"id\": \"alice\"},\n    )\n    assert not await ds.allowed(\n        action=\"execute-sql\",\n        resource=DatabaseResource(database=\"content\"),\n        actor={\"id\": \"bob\"},\n    )\n\n\n@pytest.mark.asyncio\nasync def test_database_permission():\n    config = {\n        \"databases\": {\n            \"content\": {\n                \"permissions\": {\n                    \"insert-row\": {\"id\": \"alice\"},\n                }\n            }\n        }\n    }\n    ds = await setup_datasette(config=config, databases=[\"content\"])\n\n    assert await ds.allowed(\n        action=\"insert-row\",\n        resource=TableResource(database=\"content\", table=\"repos\"),\n        actor={\"id\": \"alice\"},\n    )\n    assert not await ds.allowed(\n        action=\"insert-row\",\n        resource=TableResource(database=\"content\", table=\"repos\"),\n        actor={\"id\": \"bob\"},\n    )\n\n\n@pytest.mark.asyncio\nasync def test_table_permission():\n    config = {\n        \"databases\": {\n            \"content\": {\n                \"tables\": {\"repos\": {\"permissions\": {\"delete-row\": {\"id\": \"alice\"}}}}\n            }\n        }\n    }\n    ds = await setup_datasette(config=config, databases=[\"content\"])\n\n    assert await ds.allowed(\n        action=\"delete-row\",\n        resource=TableResource(database=\"content\", table=\"repos\"),\n        actor={\"id\": \"alice\"},\n    )\n    assert not await ds.allowed(\n        action=\"delete-row\",\n        resource=TableResource(database=\"content\", table=\"repos\"),\n        actor={\"id\": \"bob\"},\n    )\n\n\n@pytest.mark.asyncio\nasync def test_view_table_allow_block():\n    config = {\n        \"databases\": {\"content\": {\"tables\": {\"repos\": {\"allow\": {\"id\": \"alice\"}}}}}\n    }\n    ds = await setup_datasette(config=config, databases=[\"content\"])\n\n    assert await ds.allowed(\n        action=\"view-table\",\n        resource=TableResource(database=\"content\", table=\"repos\"),\n        actor={\"id\": \"alice\"},\n    )\n    assert not await ds.allowed(\n        action=\"view-table\",\n        resource=TableResource(database=\"content\", table=\"repos\"),\n        actor={\"id\": \"bob\"},\n    )\n    assert await ds.allowed(\n        action=\"view-table\",\n        resource=TableResource(database=\"content\", table=\"other\"),\n        actor={\"id\": \"bob\"},\n    )\n\n\n@pytest.mark.asyncio\nasync def test_view_table_allow_false_blocks():\n    config = {\"databases\": {\"content\": {\"tables\": {\"repos\": {\"allow\": False}}}}}\n    ds = await setup_datasette(config=config, databases=[\"content\"])\n\n    assert not await ds.allowed(\n        action=\"view-table\",\n        resource=TableResource(database=\"content\", table=\"repos\"),\n        actor={\"id\": \"alice\"},\n    )\n\n\n@pytest.mark.asyncio\nasync def test_allow_sql_blocks():\n    config = {\"allow_sql\": {\"id\": \"alice\"}}\n    ds = await setup_datasette(config=config, databases=[\"content\"])\n\n    assert await ds.allowed(\n        action=\"execute-sql\",\n        resource=DatabaseResource(database=\"content\"),\n        actor={\"id\": \"alice\"},\n    )\n    assert not await ds.allowed(\n        action=\"execute-sql\",\n        resource=DatabaseResource(database=\"content\"),\n        actor={\"id\": \"bob\"},\n    )\n\n    config = {\"databases\": {\"content\": {\"allow_sql\": {\"id\": \"bob\"}}}}\n    ds = await setup_datasette(config=config, databases=[\"content\"])\n\n    assert await ds.allowed(\n        action=\"execute-sql\",\n        resource=DatabaseResource(database=\"content\"),\n        actor={\"id\": \"bob\"},\n    )\n    assert not await ds.allowed(\n        action=\"execute-sql\",\n        resource=DatabaseResource(database=\"content\"),\n        actor={\"id\": \"alice\"},\n    )\n\n    config = {\"allow_sql\": False}\n    ds = await setup_datasette(config=config, databases=[\"content\"])\n    assert not await ds.allowed(\n        action=\"execute-sql\",\n        resource=DatabaseResource(database=\"content\"),\n        actor={\"id\": \"alice\"},\n    )\n\n\n@pytest.mark.asyncio\nasync def test_view_instance_allow_block():\n    config = {\"allow\": {\"id\": \"alice\"}}\n    ds = await setup_datasette(config=config)\n\n    assert await ds.allowed(action=\"view-instance\", actor={\"id\": \"alice\"})\n    assert not await ds.allowed(action=\"view-instance\", actor={\"id\": \"bob\"})\n"
  },
  {
    "path": "tests/test_crossdb.py",
    "content": "from datasette.cli import cli\nfrom click.testing import CliRunner\nimport urllib\nimport sqlite3\n\n\ndef test_crossdb_join(app_client_two_attached_databases_crossdb_enabled):\n    app_client = app_client_two_attached_databases_crossdb_enabled\n    sql = \"\"\"\n    select\n      'extra database' as db,\n      pk,\n      text1,\n      text2\n    from\n      [extra database].searchable\n    union all\n    select\n      'fixtures' as db,\n      pk,\n      text1,\n      text2\n    from\n      fixtures.searchable\n    \"\"\"\n    response = app_client.get(\n        \"/_memory/-/query.json?\"\n        + urllib.parse.urlencode({\"sql\": sql, \"_shape\": \"array\"})\n    )\n    assert response.status == 200\n    assert response.json == [\n        {\"db\": \"extra database\", \"pk\": 1, \"text1\": \"barry cat\", \"text2\": \"terry dog\"},\n        {\"db\": \"extra database\", \"pk\": 2, \"text1\": \"terry dog\", \"text2\": \"sara weasel\"},\n        {\"db\": \"fixtures\", \"pk\": 1, \"text1\": \"barry cat\", \"text2\": \"terry dog\"},\n        {\"db\": \"fixtures\", \"pk\": 2, \"text1\": \"terry dog\", \"text2\": \"sara weasel\"},\n    ]\n\n\ndef test_crossdb_warning_if_too_many_databases(tmp_path_factory):\n    db_dir = tmp_path_factory.mktemp(\"dbs\")\n    dbs = []\n    for i in range(11):\n        path = str(db_dir / \"db_{}.db\".format(i))\n        conn = sqlite3.connect(path)\n        conn.execute(\"vacuum\")\n        dbs.append(path)\n    runner = CliRunner()\n    result = runner.invoke(\n        cli,\n        [\n            \"serve\",\n            \"--crossdb\",\n            \"--get\",\n            \"/\",\n        ]\n        + dbs,\n        catch_exceptions=False,\n    )\n    assert (\n        \"Warning: --crossdb only works with the first 10 attached databases\"\n        in result.stderr\n    )\n\n\ndef test_crossdb_attached_database_list_display(\n    app_client_two_attached_databases_crossdb_enabled,\n):\n    app_client = app_client_two_attached_databases_crossdb_enabled\n    response = app_client.get(\"/_memory\")\n    app_client.get(\"/\")\n    for fragment in (\n        \"databases are attached to this connection\",\n        \"
  • fixtures - \",\n        '
  • extra database - \"\n\n\ndef test_redirect(custom_pages_client):\n    response = custom_pages_client.get(\"/redirect\")\n    assert response.status == 302\n    assert response.headers[\"Location\"] == \"/example\"\n\n\ndef test_redirect2(custom_pages_client):\n    response = custom_pages_client.get(\"/redirect2\")\n    assert response.status == 301\n    assert response.headers[\"Location\"] == \"/example\"\n\n\n@pytest.mark.parametrize(\n    \"path,expected\",\n    [\n        (\"/route_Sally\", \"
    Hello from Sally
    \"),\n        (\"/topic_python\", \"Topic page for python\"),\n        (\"/topic_python/info\", \"Slug: info, Topic: python\"),\n    ],\n)\ndef test_custom_route_pattern(custom_pages_client, path, expected):\n    response = custom_pages_client.get(path)\n    assert response.status == 200\n    assert response.text.strip() == expected\n\n\ndef test_custom_route_pattern_404(custom_pages_client):\n    response = custom_pages_client.get(\"/route_OhNo\")\n    assert response.status == 404\n    assert \"
    Error 404
    \" in response.text\n    assert \">Oh no /dev/null ); do\n    if [ $waiting -eq 4 ]; then\n        echo \"$server_pid does still exist, server failed to stop\"\n        cleanup\n        exit 1\n    fi\n    let waiting=waiting+1\n    sleep 1\ndone\n\n# Clean up the certificates\ncleanup\n\necho $curl_exit_code\nexit $curl_exit_code\n"
  },
  {
    "path": "tests/test_default_deny.py",
    "content": "import pytest\nfrom datasette.app import Datasette\nfrom datasette.resources import DatabaseResource, TableResource\n\n\n@pytest.mark.asyncio\nasync def test_default_deny_denies_default_permissions():\n    \"\"\"Test that default_deny=True denies default permissions\"\"\"\n    # Without default_deny, anonymous users can view instance/database/tables\n    ds_normal = Datasette()\n    await ds_normal.invoke_startup()\n\n    # Add a test database\n    db = ds_normal.add_memory_database(\"test_db_normal\")\n    await db.execute_write(\"create table test_table (id integer primary key)\")\n    await ds_normal._refresh_schemas()  # Trigger catalog refresh\n\n    # Test default behavior - anonymous user should be able to view\n    response = await ds_normal.client.get(\"/\")\n    assert response.status_code == 200\n\n    response = await ds_normal.client.get(\"/test_db_normal\")\n    assert response.status_code == 200\n\n    response = await ds_normal.client.get(\"/test_db_normal/test_table\")\n    assert response.status_code == 200\n\n    # With default_deny=True, anonymous users should be denied\n    ds_deny = Datasette(default_deny=True)\n    await ds_deny.invoke_startup()\n\n    # Add the same test database\n    db = ds_deny.add_memory_database(\"test_db_deny\")\n    await db.execute_write(\"create table test_table (id integer primary key)\")\n    await ds_deny._refresh_schemas()  # Trigger catalog refresh\n\n    # Anonymous user should be denied\n    response = await ds_deny.client.get(\"/\")\n    assert response.status_code == 403\n\n    response = await ds_deny.client.get(\"/test_db_deny\")\n    assert response.status_code == 403\n\n    response = await ds_deny.client.get(\"/test_db_deny/test_table\")\n    assert response.status_code == 403\n\n\n@pytest.mark.asyncio\nasync def test_default_deny_with_root_user():\n    \"\"\"Test that root user still has access when default_deny=True\"\"\"\n    ds = Datasette(default_deny=True)\n    ds.root_enabled = True\n    await ds.invoke_startup()\n\n    root_actor = {\"id\": \"root\"}\n\n    # Root user should have all permissions even with default_deny\n    assert await ds.allowed(action=\"view-instance\", actor=root_actor) is True\n    assert (\n        await ds.allowed(\n            action=\"view-database\",\n            actor=root_actor,\n            resource=DatabaseResource(\"test_db\"),\n        )\n        is True\n    )\n    assert (\n        await ds.allowed(\n            action=\"view-table\",\n            actor=root_actor,\n            resource=TableResource(\"test_db\", \"test_table\"),\n        )\n        is True\n    )\n    assert (\n        await ds.allowed(\n            action=\"execute-sql\", actor=root_actor, resource=DatabaseResource(\"test_db\")\n        )\n        is True\n    )\n\n\n@pytest.mark.asyncio\nasync def test_default_deny_with_config_allow():\n    \"\"\"Test that config allow rules still work with default_deny=True\"\"\"\n    ds = Datasette(default_deny=True, config={\"allow\": {\"id\": \"user1\"}})\n    await ds.invoke_startup()\n\n    # Anonymous user should be denied\n    assert await ds.allowed(action=\"view-instance\", actor=None) is False\n\n    # Authenticated user with explicit permission should have access\n    assert await ds.allowed(action=\"view-instance\", actor={\"id\": \"user1\"}) is True\n\n    # Different user should be denied\n    assert await ds.allowed(action=\"view-instance\", actor={\"id\": \"user2\"}) is False\n\n\n@pytest.mark.asyncio\nasync def test_default_deny_basic_permissions():\n    \"\"\"Test that default_deny=True denies basic permissions\"\"\"\n    ds = Datasette(default_deny=True)\n    await ds.invoke_startup()\n\n    # Anonymous user should be denied all default permissions\n    assert await ds.allowed(action=\"view-instance\", actor=None) is False\n    assert (\n        await ds.allowed(\n            action=\"view-database\", actor=None, resource=DatabaseResource(\"test_db\")\n        )\n        is False\n    )\n    assert (\n        await ds.allowed(\n            action=\"view-table\",\n            actor=None,\n            resource=TableResource(\"test_db\", \"test_table\"),\n        )\n        is False\n    )\n    assert (\n        await ds.allowed(\n            action=\"execute-sql\", actor=None, resource=DatabaseResource(\"test_db\")\n        )\n        is False\n    )\n\n    # Authenticated user without explicit permission should also be denied\n    assert await ds.allowed(action=\"view-instance\", actor={\"id\": \"user\"}) is False\n"
  },
  {
    "path": "tests/test_docs.py",
    "content": "\"\"\"\nTests to ensure certain things are documented.\n\"\"\"\n\nfrom datasette import app, utils\nfrom datasette.app import Datasette\nfrom datasette.filters import Filters\nfrom pathlib import Path\nimport pytest\nimport re\n\ndocs_path = Path(__file__).parent.parent / \"docs\"\nlabel_re = re.compile(r\"\\.\\. _([^\\s:]+):\")\n\n\ndef get_headings(content, underline=\"-\"):\n    heading_re = re.compile(r\"(\\w+)(\\([^)]*\\))?\\n\\{}+\\n\".format(underline))\n    return {h[0] for h in heading_re.findall(content)}\n\n\ndef get_labels(filename):\n    content = (docs_path / filename).read_text()\n    return set(label_re.findall(content))\n\n\n@pytest.fixture(scope=\"session\")\ndef settings_headings():\n    return get_headings((docs_path / \"settings.rst\").read_text(), \"~\")\n\n\ndef test_settings_are_documented(settings_headings, subtests):\n    for setting in app.SETTINGS:\n        with subtests.test(setting=setting.name):\n            assert setting.name in settings_headings\n\n\n@pytest.fixture(scope=\"session\")\ndef plugin_hooks_content():\n    return (docs_path / \"plugin_hooks.rst\").read_text()\n\n\ndef test_plugin_hooks_are_documented(plugin_hooks_content, subtests):\n    headings = set()\n    headings.update(get_headings(plugin_hooks_content, \"-\"))\n    headings.update(get_headings(plugin_hooks_content, \"~\"))\n    plugins = [name for name in dir(app.pm.hook) if not name.startswith(\"_\")]\n    for plugin in plugins:\n        with subtests.test(plugin=plugin):\n            assert plugin in headings\n            hook_caller = getattr(app.pm.hook, plugin)\n            arg_names = [a for a in hook_caller.spec.argnames if a != \"__multicall__\"]\n            # Check for plugin_name(arg1, arg2, arg3)\n            expected = f\"{plugin}({', '.join(arg_names)})\"\n            assert (\n                expected in plugin_hooks_content\n            ), f\"Missing from plugin hook documentation: {expected}\"\n\n\n@pytest.fixture(scope=\"session\")\ndef documented_views():\n    view_labels = set()\n    for filename in docs_path.glob(\"*.rst\"):\n        for label in get_labels(filename):\n            first_word = label.split(\"_\")[0]\n            if first_word.endswith(\"View\"):\n                view_labels.add(first_word)\n    # We deliberately don't document these:\n    view_labels.update((\"PatternPortfolioView\", \"AuthTokenView\", \"ApiExplorerView\"))\n    return view_labels\n\n\ndef test_view_classes_are_documented(documented_views, subtests):\n    view_classes = [v for v in dir(app) if v.endswith(\"View\")]\n    for view_class in view_classes:\n        with subtests.test(view_class=view_class):\n            assert view_class in documented_views\n\n\n@pytest.fixture(scope=\"session\")\ndef documented_table_filters():\n    json_api_rst = (docs_path / \"json_api.rst\").read_text()\n    section = json_api_rst.split(\".. _table_arguments:\")[-1]\n    # Lines starting with ``?column__exact= are docs for filters\n    return {\n        line.split(\"__\")[1].split(\"=\")[0]\n        for line in section.split(\"\\n\")\n        if line.startswith(\"``?column__\")\n    }\n\n\ndef test_table_filters_are_documented(documented_table_filters, subtests):\n    for f in Filters._filters:\n        with subtests.test(filter=f.key):\n            assert f.key in documented_table_filters\n\n\n@pytest.fixture(scope=\"session\")\ndef documented_fns():\n    internals_rst = (docs_path / \"internals.rst\").read_text()\n    # Any line that starts .. _internals_utils_X\n    lines = internals_rst.split(\"\\n\")\n    prefix = \".. _internals_utils_\"\n    return {\n        line.split(prefix)[1].split(\":\")[0] for line in lines if line.startswith(prefix)\n    }\n\n\ndef test_functions_marked_with_documented_are_documented(documented_fns, subtests):\n    for fn in utils.functions_marked_as_documented:\n        with subtests.test(fn=fn.__name__):\n            assert fn.__name__ in documented_fns\n\n\ndef test_rst_heading_underlines_match_title_length():\n    \"\"\"Test that RST heading underlines are the same length as their titles.\"\"\"\n    # Common RST underline characters\n    underline_chars = [\"-\", \"=\", \"~\", \"^\", \"+\", \"*\", \"#\"]\n\n    errors = []\n\n    for rst_file in docs_path.glob(\"*.rst\"):\n        content = rst_file.read_text()\n        lines = content.split(\"\\n\")\n\n        for i in range(len(lines) - 1):\n            current_line = lines[i]\n            next_line = lines[i + 1]\n\n            # Check if next line is entirely made of a single underline character\n            # and is at least 5 characters long (to avoid false positives)\n            if (\n                next_line\n                and len(next_line) >= 5\n                and len(set(next_line)) == 1\n                and next_line[0] in underline_chars\n            ):\n                # Skip if the previous line is empty (blank line before underline)\n                if not current_line:\n                    continue\n\n                # Check if this is an overline+underline style heading\n                # Look at the line before current_line to see if it's also an underline\n                if i > 0:\n                    prev_line = lines[i - 1]\n                    if (\n                        prev_line\n                        and len(prev_line) >= 5\n                        and len(set(prev_line)) == 1\n                        and prev_line[0] in underline_chars\n                        and len(prev_line) == len(next_line)\n                    ):\n                        # This is overline+underline style, skip it\n                        continue\n\n                # This is a heading underline\n                title_length = len(current_line)\n                underline_length = len(next_line)\n\n                if title_length != underline_length:\n                    errors.append(\n                        f\"{rst_file.name}:{i+1}: Title length {title_length} != underline length {underline_length}\\n\"\n                        f\"  Title: {current_line!r}\\n\"\n                        f\"  Underline: {next_line!r}\"\n                    )\n\n    if errors:\n        raise AssertionError(\n            f\"Found {len(errors)} RST heading(s) with mismatched underline length:\\n\\n\"\n            + \"\\n\\n\".join(errors)\n        )\n\n\n# Tests for testing_plugins.rst documentation\n\n# fmt: off\n# -- start test_homepage --\n@pytest.mark.asyncio\nasync def test_homepage():\n    ds = Datasette(memory=True)\n    response = await ds.client.get(\"/\")\n    html = response.text\n    assert \"
    \" in html\n# -- end test_homepage --\n\n\n# -- start test_actor_is_null --\n@pytest.mark.asyncio\nasync def test_actor_is_null():\n    ds = Datasette(memory=True)\n    response = await ds.client.get(\"/-/actor.json\")\n    assert response.json() == {\"actor\": None}\n# -- end test_actor_is_null --\n\n\n# -- start test_signed_cookie_actor --\n@pytest.mark.asyncio\nasync def test_signed_cookie_actor():\n    ds = Datasette(memory=True)\n    cookies = {\"ds_actor\": ds.client.actor_cookie({\"id\": \"root\"})}\n    response = await ds.client.get(\"/-/actor.json\", cookies=cookies)\n    assert response.json() == {\"actor\": {\"id\": \"root\"}}\n# -- end test_signed_cookie_actor --\n"
  },
  {
    "path": "tests/test_docs_plugins.py",
    "content": "# fmt: off\n# -- start datasette_with_plugin_fixture --\nfrom datasette import hookimpl\nfrom datasette.app import Datasette\nimport pytest\nimport pytest_asyncio\n\n\n@pytest_asyncio.fixture\nasync def datasette_with_plugin():\n    class TestPlugin:\n        __name__ = \"TestPlugin\"\n\n        @hookimpl\n        def register_routes(self):\n            return [\n                (r\"^/error$\", lambda: 1 / 0),\n            ]\n\n    datasette = Datasette()\n    datasette.pm.register(TestPlugin(), name=\"undo\")\n    try:\n        yield datasette\n    finally:\n        datasette.pm.unregister(name=\"undo\")\n# -- end datasette_with_plugin_fixture --\n\n\n# -- start datasette_with_plugin_test --\n@pytest.mark.asyncio\nasync def test_error(datasette_with_plugin):\n    response = await datasette_with_plugin.client.get(\"/error\")\n    assert response.status_code == 500\n# -- end datasette_with_plugin_test --\n"
  },
  {
    "path": "tests/test_facets.py",
    "content": "from datasette.app import Datasette\nfrom datasette.database import Database\nfrom datasette.facets import Facet, ColumnFacet, ArrayFacet, DateFacet\nfrom datasette.utils.asgi import Request\nfrom datasette.utils import detect_json1\nfrom .fixtures import make_app_client\nimport json\nimport pytest\n\n\n@pytest.mark.asyncio\nasync def test_column_facet_suggest(ds_client):\n    facet = ColumnFacet(\n        ds_client.ds,\n        Request.fake(\"/\"),\n        database=\"fixtures\",\n        sql=\"select * from facetable\",\n        table=\"facetable\",\n    )\n    suggestions = await facet.suggest()\n    assert [\n        {\"name\": \"created\", \"toggle_url\": \"http://localhost/?_facet=created\"},\n        {\"name\": \"planet_int\", \"toggle_url\": \"http://localhost/?_facet=planet_int\"},\n        {\"name\": \"on_earth\", \"toggle_url\": \"http://localhost/?_facet=on_earth\"},\n        {\"name\": \"state\", \"toggle_url\": \"http://localhost/?_facet=state\"},\n        {\"name\": \"_city_id\", \"toggle_url\": \"http://localhost/?_facet=_city_id\"},\n        {\n            \"name\": \"_neighborhood\",\n            \"toggle_url\": \"http://localhost/?_facet=_neighborhood\",\n        },\n        {\"name\": \"tags\", \"toggle_url\": \"http://localhost/?_facet=tags\"},\n        {\n            \"name\": \"complex_array\",\n            \"toggle_url\": \"http://localhost/?_facet=complex_array\",\n        },\n    ] == suggestions\n\n\n@pytest.mark.asyncio\nasync def test_column_facet_suggest_skip_if_already_selected(ds_client):\n    facet = ColumnFacet(\n        ds_client.ds,\n        Request.fake(\"/?_facet=planet_int&_facet=on_earth\"),\n        database=\"fixtures\",\n        sql=\"select * from facetable\",\n        table=\"facetable\",\n    )\n    suggestions = await facet.suggest()\n    assert [\n        {\n            \"name\": \"created\",\n            \"toggle_url\": \"http://localhost/?_facet=planet_int&_facet=on_earth&_facet=created\",\n        },\n        {\n            \"name\": \"state\",\n            \"toggle_url\": \"http://localhost/?_facet=planet_int&_facet=on_earth&_facet=state\",\n        },\n        {\n            \"name\": \"_city_id\",\n            \"toggle_url\": \"http://localhost/?_facet=planet_int&_facet=on_earth&_facet=_city_id\",\n        },\n        {\n            \"name\": \"_neighborhood\",\n            \"toggle_url\": \"http://localhost/?_facet=planet_int&_facet=on_earth&_facet=_neighborhood\",\n        },\n        {\n            \"name\": \"tags\",\n            \"toggle_url\": \"http://localhost/?_facet=planet_int&_facet=on_earth&_facet=tags\",\n        },\n        {\n            \"name\": \"complex_array\",\n            \"toggle_url\": \"http://localhost/?_facet=planet_int&_facet=on_earth&_facet=complex_array\",\n        },\n    ] == suggestions\n\n\n@pytest.mark.asyncio\nasync def test_column_facet_suggest_skip_if_enabled_by_metadata(ds_client):\n    facet = ColumnFacet(\n        ds_client.ds,\n        Request.fake(\"/\"),\n        database=\"fixtures\",\n        sql=\"select * from facetable\",\n        table=\"facetable\",\n        table_config={\"facets\": [\"_city_id\"]},\n    )\n    suggestions = [s[\"name\"] for s in await facet.suggest()]\n    assert [\n        \"created\",\n        \"planet_int\",\n        \"on_earth\",\n        \"state\",\n        \"_neighborhood\",\n        \"tags\",\n        \"complex_array\",\n    ] == suggestions\n\n\n@pytest.mark.asyncio\nasync def test_column_facet_results(ds_client):\n    facet = ColumnFacet(\n        ds_client.ds,\n        Request.fake(\"/?_facet=_city_id\"),\n        database=\"fixtures\",\n        sql=\"select * from facetable\",\n        table=\"facetable\",\n    )\n    buckets, timed_out = await facet.facet_results()\n    assert [] == timed_out\n    assert [\n        {\n            \"name\": \"_city_id\",\n            \"type\": \"column\",\n            \"hideable\": True,\n            \"toggle_url\": \"/\",\n            \"results\": [\n                {\n                    \"value\": 1,\n                    \"label\": \"San Francisco\",\n                    \"count\": 6,\n                    \"toggle_url\": \"http://localhost/?_facet=_city_id&_city_id__exact=1\",\n                    \"selected\": False,\n                },\n                {\n                    \"value\": 2,\n                    \"label\": \"Los Angeles\",\n                    \"count\": 4,\n                    \"toggle_url\": \"http://localhost/?_facet=_city_id&_city_id__exact=2\",\n                    \"selected\": False,\n                },\n                {\n                    \"value\": 3,\n                    \"label\": \"Detroit\",\n                    \"count\": 4,\n                    \"toggle_url\": \"http://localhost/?_facet=_city_id&_city_id__exact=3\",\n                    \"selected\": False,\n                },\n                {\n                    \"value\": 4,\n                    \"label\": \"Memnonia\",\n                    \"count\": 1,\n                    \"toggle_url\": \"http://localhost/?_facet=_city_id&_city_id__exact=4\",\n                    \"selected\": False,\n                },\n            ],\n            \"truncated\": False,\n        }\n    ] == buckets\n\n\n@pytest.mark.asyncio\nasync def test_column_facet_results_column_starts_with_underscore(ds_client):\n    facet = ColumnFacet(\n        ds_client.ds,\n        Request.fake(\"/?_facet=_neighborhood\"),\n        database=\"fixtures\",\n        sql=\"select * from facetable\",\n        table=\"facetable\",\n    )\n    buckets, timed_out = await facet.facet_results()\n    assert [] == timed_out\n    assert buckets == [\n        {\n            \"name\": \"_neighborhood\",\n            \"type\": \"column\",\n            \"hideable\": True,\n            \"toggle_url\": \"/\",\n            \"results\": [\n                {\n                    \"value\": \"Downtown\",\n                    \"label\": \"Downtown\",\n                    \"count\": 2,\n                    \"toggle_url\": \"http://localhost/?_facet=_neighborhood&_neighborhood__exact=Downtown\",\n                    \"selected\": False,\n                },\n                {\n                    \"value\": \"Arcadia Planitia\",\n                    \"label\": \"Arcadia Planitia\",\n                    \"count\": 1,\n                    \"toggle_url\": \"http://localhost/?_facet=_neighborhood&_neighborhood__exact=Arcadia+Planitia\",\n                    \"selected\": False,\n                },\n                {\n                    \"value\": \"Bernal Heights\",\n                    \"label\": \"Bernal Heights\",\n                    \"count\": 1,\n                    \"toggle_url\": \"http://localhost/?_facet=_neighborhood&_neighborhood__exact=Bernal+Heights\",\n                    \"selected\": False,\n                },\n                {\n                    \"value\": \"Corktown\",\n                    \"label\": \"Corktown\",\n                    \"count\": 1,\n                    \"toggle_url\": \"http://localhost/?_facet=_neighborhood&_neighborhood__exact=Corktown\",\n                    \"selected\": False,\n                },\n                {\n                    \"value\": \"Dogpatch\",\n                    \"label\": \"Dogpatch\",\n                    \"count\": 1,\n                    \"toggle_url\": \"http://localhost/?_facet=_neighborhood&_neighborhood__exact=Dogpatch\",\n                    \"selected\": False,\n                },\n                {\n                    \"value\": \"Greektown\",\n                    \"label\": \"Greektown\",\n                    \"count\": 1,\n                    \"toggle_url\": \"http://localhost/?_facet=_neighborhood&_neighborhood__exact=Greektown\",\n                    \"selected\": False,\n                },\n                {\n                    \"value\": \"Hayes Valley\",\n                    \"label\": \"Hayes Valley\",\n                    \"count\": 1,\n                    \"toggle_url\": \"http://localhost/?_facet=_neighborhood&_neighborhood__exact=Hayes+Valley\",\n                    \"selected\": False,\n                },\n                {\n                    \"value\": \"Hollywood\",\n                    \"label\": \"Hollywood\",\n                    \"count\": 1,\n                    \"toggle_url\": \"http://localhost/?_facet=_neighborhood&_neighborhood__exact=Hollywood\",\n                    \"selected\": False,\n                },\n                {\n                    \"value\": \"Koreatown\",\n                    \"label\": \"Koreatown\",\n                    \"count\": 1,\n                    \"toggle_url\": \"http://localhost/?_facet=_neighborhood&_neighborhood__exact=Koreatown\",\n                    \"selected\": False,\n                },\n                {\n                    \"value\": \"Los Feliz\",\n                    \"label\": \"Los Feliz\",\n                    \"count\": 1,\n                    \"toggle_url\": \"http://localhost/?_facet=_neighborhood&_neighborhood__exact=Los+Feliz\",\n                    \"selected\": False,\n                },\n                {\n                    \"value\": \"Mexicantown\",\n                    \"label\": \"Mexicantown\",\n                    \"count\": 1,\n                    \"toggle_url\": \"http://localhost/?_facet=_neighborhood&_neighborhood__exact=Mexicantown\",\n                    \"selected\": False,\n                },\n                {\n                    \"value\": \"Mission\",\n                    \"label\": \"Mission\",\n                    \"count\": 1,\n                    \"toggle_url\": \"http://localhost/?_facet=_neighborhood&_neighborhood__exact=Mission\",\n                    \"selected\": False,\n                },\n                {\n                    \"value\": \"SOMA\",\n                    \"label\": \"SOMA\",\n                    \"count\": 1,\n                    \"toggle_url\": \"http://localhost/?_facet=_neighborhood&_neighborhood__exact=SOMA\",\n                    \"selected\": False,\n                },\n                {\n                    \"value\": \"Tenderloin\",\n                    \"label\": \"Tenderloin\",\n                    \"count\": 1,\n                    \"toggle_url\": \"http://localhost/?_facet=_neighborhood&_neighborhood__exact=Tenderloin\",\n                    \"selected\": False,\n                },\n            ],\n            \"truncated\": False,\n        }\n    ]\n\n\n@pytest.mark.asyncio\nasync def test_column_facet_from_metadata_cannot_be_hidden(ds_client):\n    facet = ColumnFacet(\n        ds_client.ds,\n        Request.fake(\"/\"),\n        database=\"fixtures\",\n        sql=\"select * from facetable\",\n        table=\"facetable\",\n        table_config={\"facets\": [\"_city_id\"]},\n    )\n    buckets, timed_out = await facet.facet_results()\n    assert [] == timed_out\n    assert [\n        {\n            \"name\": \"_city_id\",\n            \"type\": \"column\",\n            \"hideable\": False,\n            \"toggle_url\": \"/\",\n            \"results\": [\n                {\n                    \"value\": 1,\n                    \"label\": \"San Francisco\",\n                    \"count\": 6,\n                    \"toggle_url\": \"http://localhost/?_city_id__exact=1\",\n                    \"selected\": False,\n                },\n                {\n                    \"value\": 2,\n                    \"label\": \"Los Angeles\",\n                    \"count\": 4,\n                    \"toggle_url\": \"http://localhost/?_city_id__exact=2\",\n                    \"selected\": False,\n                },\n                {\n                    \"value\": 3,\n                    \"label\": \"Detroit\",\n                    \"count\": 4,\n                    \"toggle_url\": \"http://localhost/?_city_id__exact=3\",\n                    \"selected\": False,\n                },\n                {\n                    \"value\": 4,\n                    \"label\": \"Memnonia\",\n                    \"count\": 1,\n                    \"toggle_url\": \"http://localhost/?_city_id__exact=4\",\n                    \"selected\": False,\n                },\n            ],\n            \"truncated\": False,\n        }\n    ] == buckets\n\n\n@pytest.mark.asyncio\n@pytest.mark.skipif(not detect_json1(), reason=\"Requires the SQLite json1 module\")\nasync def test_array_facet_suggest(ds_client):\n    facet = ArrayFacet(\n        ds_client.ds,\n        Request.fake(\"/\"),\n        database=\"fixtures\",\n        sql=\"select * from facetable\",\n        table=\"facetable\",\n    )\n    suggestions = await facet.suggest()\n    assert [\n        {\n            \"name\": \"tags\",\n            \"type\": \"array\",\n            \"toggle_url\": \"http://localhost/?_facet_array=tags\",\n        }\n    ] == suggestions\n\n\n@pytest.mark.asyncio\n@pytest.mark.skipif(not detect_json1(), reason=\"Requires the SQLite json1 module\")\nasync def test_array_facet_suggest_not_if_all_empty_arrays(ds_client):\n    facet = ArrayFacet(\n        ds_client.ds,\n        Request.fake(\"/\"),\n        database=\"fixtures\",\n        sql=\"select * from facetable where tags = '[]'\",\n        table=\"facetable\",\n    )\n    suggestions = await facet.suggest()\n    assert [] == suggestions\n\n\n@pytest.mark.asyncio\n@pytest.mark.skipif(not detect_json1(), reason=\"Requires the SQLite json1 module\")\nasync def test_array_facet_results(ds_client):\n    facet = ArrayFacet(\n        ds_client.ds,\n        Request.fake(\"/?_facet_array=tags\"),\n        database=\"fixtures\",\n        sql=\"select * from facetable\",\n        table=\"facetable\",\n    )\n    buckets, timed_out = await facet.facet_results()\n    assert [] == timed_out\n    assert [\n        {\n            \"name\": \"tags\",\n            \"type\": \"array\",\n            \"results\": [\n                {\n                    \"value\": \"tag1\",\n                    \"label\": \"tag1\",\n                    \"count\": 2,\n                    \"toggle_url\": \"http://localhost/?_facet_array=tags&tags__arraycontains=tag1\",\n                    \"selected\": False,\n                },\n                {\n                    \"value\": \"tag2\",\n                    \"label\": \"tag2\",\n                    \"count\": 1,\n                    \"toggle_url\": \"http://localhost/?_facet_array=tags&tags__arraycontains=tag2\",\n                    \"selected\": False,\n                },\n                {\n                    \"value\": \"tag3\",\n                    \"label\": \"tag3\",\n                    \"count\": 1,\n                    \"toggle_url\": \"http://localhost/?_facet_array=tags&tags__arraycontains=tag3\",\n                    \"selected\": False,\n                },\n            ],\n            \"hideable\": True,\n            \"toggle_url\": \"/\",\n            \"truncated\": False,\n        }\n    ] == buckets\n\n\n@pytest.mark.asyncio\n@pytest.mark.skipif(not detect_json1(), reason=\"Requires the SQLite json1 module\")\nasync def test_array_facet_handle_duplicate_tags():\n    ds = Datasette([], memory=True)\n    db = ds.add_database(Database(ds, memory_name=\"test_array_facet\"))\n    await db.execute_write(\"create table otters(name text, tags text)\")\n    for name, tags in (\n        (\"Charles\", [\"friendly\", \"cunning\", \"friendly\"]),\n        (\"Shaun\", [\"cunning\", \"empathetic\", \"friendly\"]),\n        (\"Tracy\", [\"empathetic\", \"eager\"]),\n    ):\n        await db.execute_write(\n            \"insert into otters (name, tags) values (?, ?)\", [name, json.dumps(tags)]\n        )\n\n    response = await ds.client.get(\"/test_array_facet/otters.json?_facet_array=tags\")\n    assert response.json()[\"facet_results\"][\"results\"][\"tags\"] == {\n        \"name\": \"tags\",\n        \"type\": \"array\",\n        \"results\": [\n            {\n                \"value\": \"cunning\",\n                \"label\": \"cunning\",\n                \"count\": 2,\n                \"toggle_url\": \"http://localhost/test_array_facet/otters.json?_facet_array=tags&tags__arraycontains=cunning\",\n                \"selected\": False,\n            },\n            {\n                \"value\": \"empathetic\",\n                \"label\": \"empathetic\",\n                \"count\": 2,\n                \"toggle_url\": \"http://localhost/test_array_facet/otters.json?_facet_array=tags&tags__arraycontains=empathetic\",\n                \"selected\": False,\n            },\n            {\n                \"value\": \"friendly\",\n                \"label\": \"friendly\",\n                \"count\": 2,\n                \"toggle_url\": \"http://localhost/test_array_facet/otters.json?_facet_array=tags&tags__arraycontains=friendly\",\n                \"selected\": False,\n            },\n            {\n                \"value\": \"eager\",\n                \"label\": \"eager\",\n                \"count\": 1,\n                \"toggle_url\": \"http://localhost/test_array_facet/otters.json?_facet_array=tags&tags__arraycontains=eager\",\n                \"selected\": False,\n            },\n        ],\n        \"hideable\": True,\n        \"toggle_url\": \"/test_array_facet/otters.json\",\n        \"truncated\": False,\n    }\n\n\n@pytest.mark.asyncio\nasync def test_date_facet_results(ds_client):\n    facet = DateFacet(\n        ds_client.ds,\n        Request.fake(\"/?_facet_date=created\"),\n        database=\"fixtures\",\n        sql=\"select * from facetable\",\n        table=\"facetable\",\n    )\n    buckets, timed_out = await facet.facet_results()\n    assert [] == timed_out\n    assert [\n        {\n            \"name\": \"created\",\n            \"type\": \"date\",\n            \"results\": [\n                {\n                    \"value\": \"2019-01-14\",\n                    \"label\": \"2019-01-14\",\n                    \"count\": 4,\n                    \"toggle_url\": \"http://localhost/?_facet_date=created&created__date=2019-01-14\",\n                    \"selected\": False,\n                },\n                {\n                    \"value\": \"2019-01-15\",\n                    \"label\": \"2019-01-15\",\n                    \"count\": 4,\n                    \"toggle_url\": \"http://localhost/?_facet_date=created&created__date=2019-01-15\",\n                    \"selected\": False,\n                },\n                {\n                    \"value\": \"2019-01-17\",\n                    \"label\": \"2019-01-17\",\n                    \"count\": 4,\n                    \"toggle_url\": \"http://localhost/?_facet_date=created&created__date=2019-01-17\",\n                    \"selected\": False,\n                },\n                {\n                    \"value\": \"2019-01-16\",\n                    \"label\": \"2019-01-16\",\n                    \"count\": 3,\n                    \"toggle_url\": \"http://localhost/?_facet_date=created&created__date=2019-01-16\",\n                    \"selected\": False,\n                },\n            ],\n            \"hideable\": True,\n            \"toggle_url\": \"/\",\n            \"truncated\": False,\n        }\n    ] == buckets\n\n\n@pytest.mark.asyncio\nasync def test_json_array_with_blanks_and_nulls():\n    ds = Datasette([], memory=True)\n    db = ds.add_database(Database(ds, memory_name=\"test_json_array\"))\n    await db.execute_write(\"create table foo(json_column text)\")\n    for value in ('[\"a\", \"b\", \"c\"]', '[\"a\", \"b\"]', \"\", None):\n        await db.execute_write(\"insert into foo (json_column) values (?)\", [value])\n    response = await ds.client.get(\"/test_json_array/foo.json?_extra=suggested_facets\")\n    data = response.json()\n    assert data[\"suggested_facets\"] == [\n        {\n            \"name\": \"json_column\",\n            \"type\": \"array\",\n            \"toggle_url\": \"http://localhost/test_json_array/foo.json?_extra=suggested_facets&_facet_array=json_column\",\n        }\n    ]\n\n\n@pytest.mark.asyncio\nasync def test_facet_size():\n    ds = Datasette([], memory=True, settings={\"max_returned_rows\": 50})\n    db = ds.add_database(Database(ds, memory_name=\"test_facet_size\"))\n    await db.execute_write(\"create table neighbourhoods(city text, neighbourhood text)\")\n    for i in range(1, 51):\n        for j in range(1, 4):\n            await db.execute_write(\n                \"insert into neighbourhoods (city, neighbourhood) values (?, ?)\",\n                [\"City {}\".format(i), \"Neighbourhood {}\".format(j)],\n            )\n    response = await ds.client.get(\n        \"/test_facet_size/neighbourhoods.json?_extra=suggested_facets\"\n    )\n    data = response.json()\n    assert data[\"suggested_facets\"] == [\n        {\n            \"name\": \"neighbourhood\",\n            \"toggle_url\": \"http://localhost/test_facet_size/neighbourhoods.json?_extra=suggested_facets&_facet=neighbourhood\",\n        }\n    ]\n    # Bump up _facet_size= to suggest city too\n    response2 = await ds.client.get(\n        \"/test_facet_size/neighbourhoods.json?_facet_size=50&_extra=suggested_facets\"\n    )\n    data2 = response2.json()\n    assert sorted(data2[\"suggested_facets\"], key=lambda f: f[\"name\"]) == [\n        {\n            \"name\": \"city\",\n            \"toggle_url\": \"http://localhost/test_facet_size/neighbourhoods.json?_facet_size=50&_extra=suggested_facets&_facet=city\",\n        },\n        {\n            \"name\": \"neighbourhood\",\n            \"toggle_url\": \"http://localhost/test_facet_size/neighbourhoods.json?_facet_size=50&_extra=suggested_facets&_facet=neighbourhood\",\n        },\n    ]\n    # Facet by city should return expected number of results\n    response3 = await ds.client.get(\n        \"/test_facet_size/neighbourhoods.json?_facet_size=50&_facet=city\"\n    )\n    data3 = response3.json()\n    assert len(data3[\"facet_results\"][\"results\"][\"city\"][\"results\"]) == 50\n    # Reduce max_returned_rows and check that it's respected\n    ds._settings[\"max_returned_rows\"] = 20\n    response4 = await ds.client.get(\n        \"/test_facet_size/neighbourhoods.json?_facet_size=50&_facet=city\"\n    )\n    data4 = response4.json()\n    assert len(data4[\"facet_results\"][\"results\"][\"city\"][\"results\"]) == 20\n    # Test _facet_size=max\n    response5 = await ds.client.get(\n        \"/test_facet_size/neighbourhoods.json?_facet_size=max&_facet=city\"\n    )\n    data5 = response5.json()\n    assert len(data5[\"facet_results\"][\"results\"][\"city\"][\"results\"]) == 20\n    # Now try messing with facet_size in the table metadata\n    orig_config = ds.config\n    try:\n        ds.config = {\n            \"databases\": {\n                \"test_facet_size\": {\"tables\": {\"neighbourhoods\": {\"facet_size\": 6}}}\n            }\n        }\n        response6 = await ds.client.get(\n            \"/test_facet_size/neighbourhoods.json?_facet=city\"\n        )\n        data6 = response6.json()\n        assert len(data6[\"facet_results\"][\"results\"][\"city\"][\"results\"]) == 6\n        # Setting it to max bumps it up to 50 again\n        ds.config[\"databases\"][\"test_facet_size\"][\"tables\"][\"neighbourhoods\"][\n            \"facet_size\"\n        ] = \"max\"\n        data7 = (\n            await ds.client.get(\"/test_facet_size/neighbourhoods.json?_facet=city\")\n        ).json()\n        assert len(data7[\"facet_results\"][\"results\"][\"city\"][\"results\"]) == 20\n    finally:\n        ds.config = orig_config\n\n\ndef test_other_types_of_facet_in_metadata():\n    with make_app_client(\n        metadata={\n            \"databases\": {\n                \"fixtures\": {\n                    \"tables\": {\n                        \"facetable\": {\n                            \"facets\": [\"state\", {\"array\": \"tags\"}, {\"date\": \"created\"}]\n                        }\n                    }\n                }\n            }\n        }\n    ) as client:\n        response = client.get(\"/fixtures/facetable\")\n        fragments = (\n            \"state\\n\",\n            \"tags (array)\\n\",\n            \"created (date)\\n\",\n        )\n        for fragment in fragments:\n            assert fragment in response.text\n        # Verify they appear in the metadata-defined order\n        positions = [response.text.index(f) for f in fragments]\n        assert positions == sorted(\n            positions\n        ), \"Facets should appear in metadata-defined order\"\n\n\ndef test_metadata_facet_ordering():\n    with make_app_client(\n        metadata={\n            \"databases\": {\n                \"fixtures\": {\n                    \"tables\": {\n                        \"facetable\": {\n                            \"facets\": [\"state\", {\"array\": \"tags\"}, {\"date\": \"created\"}]\n                        }\n                    }\n                }\n            }\n        }\n    ) as client:\n        # JSON response should have facets in the metadata-defined order\n        response = client.get(\"/fixtures/facetable.json?_extra=sorted_facet_results\")\n        data = response.json\n        facet_names = [f[\"name\"] for f in data[\"sorted_facet_results\"]]\n        assert facet_names == [\"state\", \"tags\", \"created\"]\n\n        # With an additional request-based facet, metadata facets come first\n        # in their defined order, followed by request-based facets\n        response2 = client.get(\n            \"/fixtures/facetable.json?_extra=sorted_facet_results&_facet=_city_id\"\n        )\n        data2 = response2.json\n        facet_names2 = [f[\"name\"] for f in data2[\"sorted_facet_results\"]]\n        assert facet_names2 == [\"state\", \"tags\", \"created\", \"_city_id\"]\n\n\n@pytest.mark.asyncio\nasync def test_conflicting_facet_names_json(ds_client):\n    response = await ds_client.get(\n        \"/fixtures/facetable.json?_facet=created&_facet_date=created\"\n        \"&_facet=tags&_facet_array=tags\"\n    )\n    assert set(response.json()[\"facet_results\"][\"results\"].keys()) == {\n        \"created\",\n        \"tags\",\n        \"created_2\",\n        \"tags_2\",\n    }\n\n\n@pytest.mark.asyncio\nasync def test_facet_against_in_memory_database():\n    ds = Datasette()\n    db = ds.add_memory_database(\"mem\")\n    await db.execute_write(\n        \"create table t (id integer primary key, name text, name2 text)\"\n    )\n    to_insert = [{\"name\": \"one\", \"name2\": \"1\"} for _ in range(800)] + [\n        {\"name\": \"two\", \"name2\": \"2\"} for _ in range(300)\n    ]\n    await db.execute_write_many(\n        \"insert into t (name, name2) values (:name, :name2)\", to_insert\n    )\n    response1 = await ds.client.get(\"/mem/t\")\n    assert response1.status_code == 200\n    response2 = await ds.client.get(\"/mem/t?_facet=name&_facet=name2\")\n    assert response2.status_code == 200\n\n\n@pytest.mark.asyncio\nasync def test_facet_only_considers_first_x_rows():\n    # This test works by manually fiddling with Facet.suggest_consider\n    ds = Datasette()\n    original_suggest_consider = Facet.suggest_consider\n    try:\n        Facet.suggest_consider = 40\n        db = ds.add_memory_database(\"test_facet_only_x_rows\")\n        await db.execute_write(\"create table t (id integer primary key, col text)\")\n        # First 50 rows make it look like col and col_json should be faceted\n        to_insert = [{\"col\": \"one\" if i % 2 else \"two\"} for i in range(50)]\n        await db.execute_write_many(\"insert into t (col) values (:col)\", to_insert)\n        # Next 50 break that assumption\n        to_insert2 = [{\"col\": f\"x{i}\"} for i in range(50)]\n        await db.execute_write_many(\"insert into t (col) values (:col)\", to_insert2)\n        response = await ds.client.get(\n            \"/test_facet_only_x_rows/t.json?_extra=suggested_facets\"\n        )\n        data = response.json()\n        assert data[\"suggested_facets\"] == [\n            {\n                \"name\": \"col\",\n                \"toggle_url\": \"http://localhost/test_facet_only_x_rows/t.json?_extra=suggested_facets&_facet=col\",\n            }\n        ]\n        # But if we set suggest_consider to 100 they are not suggested\n        Facet.suggest_consider = 100\n        response2 = await ds.client.get(\n            \"/test_facet_only_x_rows/t.json?_extra=suggested_facets\"\n        )\n        data2 = response2.json()\n        assert data2[\"suggested_facets\"] == []\n    finally:\n        Facet.suggest_consider = original_suggest_consider\n"
  },
  {
    "path": "tests/test_filters.py",
    "content": "from datasette.filters import Filters, through_filters, where_filters, search_filters\nfrom datasette.utils.asgi import Request\nimport pytest\n\n\n@pytest.mark.parametrize(\n    \"args,expected_where,expected_params\",\n    [\n        (((\"name_english__contains\", \"foo\"),), ['\"name_english\" like :p0'], [\"%foo%\"]),\n        (\n            ((\"name_english__notcontains\", \"foo\"),),\n            ['\"name_english\" not like :p0'],\n            [\"%foo%\"],\n        ),\n        (\n            ((\"foo\", \"bar\"), (\"bar__contains\", \"baz\")),\n            ['\"bar\" like :p0', '\"foo\" = :p1'],\n            [\"%baz%\", \"bar\"],\n        ),\n        (\n            ((\"foo__startswith\", \"bar\"), (\"bar__endswith\", \"baz\")),\n            ['\"bar\" like :p0', '\"foo\" like :p1'],\n            [\"%baz\", \"bar%\"],\n        ),\n        (\n            ((\"foo__lt\", \"1\"), (\"bar__gt\", \"2\"), (\"baz__gte\", \"3\"), (\"bax__lte\", \"4\")),\n            ['\"bar\" > :p0', '\"bax\" <= :p1', '\"baz\" >= :p2', '\"foo\" < :p3'],\n            [2, 4, 3, 1],\n        ),\n        (\n            ((\"foo__like\", \"2%2\"), (\"zax__glob\", \"3*\")),\n            ['\"foo\" like :p0', '\"zax\" glob :p1'],\n            [\"2%2\", \"3*\"],\n        ),\n        # Multiple like arguments:\n        (\n            ((\"foo__like\", \"2%2\"), (\"foo__like\", \"3%3\")),\n            ['\"foo\" like :p0', '\"foo\" like :p1'],\n            [\"2%2\", \"3%3\"],\n        ),\n        # notlike:\n        (\n            ((\"foo__notlike\", \"2%2\"),),\n            ['\"foo\" not like :p0'],\n            [\"2%2\"],\n        ),\n        (\n            ((\"foo__isnull\", \"1\"), (\"baz__isnull\", \"1\"), (\"bar__gt\", \"10\")),\n            ['\"bar\" > :p0', '\"baz\" is null', '\"foo\" is null'],\n            [10],\n        ),\n        (((\"foo__in\", \"1,2,3\"),), [\"foo in (:p0, :p1, :p2)\"], [\"1\", \"2\", \"3\"]),\n        # date\n        (((\"foo__date\", \"1988-01-01\"),), ['date(\"foo\") = :p0'], [\"1988-01-01\"]),\n        # JSON array variants of __in (useful for unexpected characters)\n        (((\"foo__in\", \"[1,2,3]\"),), [\"foo in (:p0, :p1, :p2)\"], [1, 2, 3]),\n        (\n            ((\"foo__in\", '[\"dog,cat\", \"cat[dog]\"]'),),\n            [\"foo in (:p0, :p1)\"],\n            [\"dog,cat\", \"cat[dog]\"],\n        ),\n        # Not in, and JSON array not in\n        (((\"foo__notin\", \"1,2,3\"),), [\"foo not in (:p0, :p1, :p2)\"], [\"1\", \"2\", \"3\"]),\n        (((\"foo__notin\", \"[1,2,3]\"),), [\"foo not in (:p0, :p1, :p2)\"], [1, 2, 3]),\n        # JSON arraycontains, arraynotcontains\n        (\n            ((\"Availability+Info__arraycontains\", \"yes\"),),\n            [\":p0 in (select value from json_each([table].[Availability+Info]))\"],\n            [\"yes\"],\n        ),\n        (\n            ((\"Availability+Info__arraynotcontains\", \"yes\"),),\n            [\":p0 not in (select value from json_each([table].[Availability+Info]))\"],\n            [\"yes\"],\n        ),\n    ],\n)\ndef test_build_where(args, expected_where, expected_params):\n    f = Filters(sorted(args))\n    sql_bits, actual_params = f.build_where_clauses(\"table\")\n    assert expected_where == sql_bits\n    assert {f\"p{i}\": param for i, param in enumerate(expected_params)} == actual_params\n\n\n@pytest.mark.asyncio\nasync def test_through_filters_from_request(ds_client):\n    request = Request.fake(\n        '/?_through={\"table\":\"roadside_attraction_characteristics\",\"column\":\"characteristic_id\",\"value\":\"1\"}'\n    )\n    filter_args = await through_filters(\n        request=request,\n        datasette=ds_client.ds,\n        table=\"roadside_attractions\",\n        database=\"fixtures\",\n    )()\n    assert filter_args.where_clauses == [\n        \"pk in (select attraction_id from roadside_attraction_characteristics where characteristic_id = :p0)\"\n    ]\n    assert filter_args.params == {\"p0\": \"1\"}\n    assert filter_args.human_descriptions == [\n        'roadside_attraction_characteristics.characteristic_id = \"1\"'\n    ]\n    assert filter_args.extra_context == {}\n\n\n@pytest.mark.asyncio\nasync def test_where_filters_from_request(ds_client):\n    await ds_client.ds.invoke_startup()\n    request = Request.fake(\"/?_where=pk+>+3\")\n    filter_args = await where_filters(\n        request=request,\n        datasette=ds_client.ds,\n        database=\"fixtures\",\n    )()\n    assert filter_args.where_clauses == [\"pk > 3\"]\n    assert filter_args.params == {}\n    assert filter_args.human_descriptions == []\n    assert filter_args.extra_context == {\n        \"extra_wheres_for_ui\": [{\"text\": \"pk > 3\", \"remove_url\": \"/\"}]\n    }\n\n\n@pytest.mark.asyncio\nasync def test_search_filters_from_request(ds_client):\n    request = Request.fake(\"/?_search=bobcat\")\n    filter_args = await search_filters(\n        request=request,\n        datasette=ds_client.ds,\n        database=\"fixtures\",\n        table=\"searchable\",\n    )()\n    assert filter_args.where_clauses == [\n        \"rowid in (select rowid from searchable_fts where searchable_fts match escape_fts(:search))\"\n    ]\n    assert filter_args.params == {\"search\": \"bobcat\"}\n    assert filter_args.human_descriptions == ['search matches \"bobcat\"']\n    assert filter_args.extra_context == {\"supports_search\": True, \"search\": \"bobcat\"}\n"
  },
  {
    "path": "tests/test_html.py",
    "content": "from bs4 import BeautifulSoup as Soup\nfrom datasette.app import Datasette\nfrom datasette.utils import allowed_pragmas\nfrom .fixtures import make_app_client\nfrom .utils import assert_footer_links, inner_html\nimport copy\nimport json\nimport pathlib\nimport pytest\nimport re\nimport urllib.parse\n\n\ndef test_homepage(app_client_two_attached_databases):\n    response = app_client_two_attached_databases.get(\"/\")\n    assert response.status_code == 200\n    assert \"text/html; charset=utf-8\" == response.headers[\"content-type\"]\n    # Should have a html lang=\"en\" attribute\n    assert '' in response.text\n    soup = Soup(response.content, \"html.parser\")\n    assert \"Datasette Fixtures\" == soup.find(\"h1\").text\n    assert (\n        \"An example SQLite database demonstrating Datasette. Sign in as root user\"\n        == soup.select(\".metadata-description\")[0].text.strip()\n    )\n    # Should be two attached databases\n    assert [\n        {\"href\": \"/extra+database\", \"text\": \"extra database\"},\n        {\"href\": \"/fixtures\", \"text\": \"fixtures\"},\n    ] == [{\"href\": a[\"href\"], \"text\": a.text.strip()} for a in soup.select(\"h2 a\")]\n    # Database should show count text and attached tables\n    h2 = soup.select(\"h2\")[0]\n    assert \"extra database\" == h2.text.strip()\n    counts_p, links_p = h2.find_all_next(\"p\")[:2]\n    assert (\n        \"2 rows in 1 table, 5 rows in 4 hidden tables, 1 view\" == counts_p.text.strip()\n    )\n    # We should only show visible, not hidden tables here:\n    table_links = [\n        {\"href\": a[\"href\"], \"text\": a.text.strip()} for a in links_p.find_all(\"a\")\n    ]\n    assert [\n        {\"href\": r\"/extra+database/searchable\", \"text\": \"searchable\"},\n        {\"href\": r\"/extra+database/searchable_view\", \"text\": \"searchable_view\"},\n    ] == table_links\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\"path\", (\"/\", \"/-/\"))\nasync def test_homepage_alternative_location(path, tmp_path_factory):\n    template_dir = tmp_path_factory.mktemp(\"templates\")\n    (template_dir / \"index.html\").write_text(\"Custom homepage\", \"utf-8\")\n    datasette = Datasette(template_dir=str(template_dir))\n    response = await datasette.client.get(path)\n    assert response.status_code == 200\n    html = response.text\n    if path == \"/\":\n        assert html == \"Custom homepage\"\n    else:\n        assert '' in html\n\n\n@pytest.mark.asyncio\nasync def test_homepage_alternative_redirect(ds_client):\n    response = await ds_client.get(\"/-\")\n    assert response.status_code == 301\n\n\n@pytest.mark.asyncio\nasync def test_http_head(ds_client):\n    response = await ds_client.head(\"/\")\n    assert response.status_code == 200\n\n\n@pytest.mark.asyncio\nasync def test_homepage_options(ds_client):\n    response = await ds_client.options(\"/\")\n    assert response.status_code == 200\n    assert response.text == \"ok\"\n\n\n@pytest.mark.asyncio\nasync def test_favicon(ds_client):\n    response = await ds_client.get(\"/favicon.ico\")\n    assert response.status_code == 200\n    assert response.headers[\"cache-control\"] == \"max-age=3600, immutable, public\"\n    assert int(response.headers[\"content-length\"]) > 100\n    assert response.headers[\"content-type\"] == \"image/png\"\n\n\n@pytest.mark.asyncio\nasync def test_static(ds_client):\n    response = await ds_client.get(\"/-/static/app2.css\")\n    assert response.status_code == 404\n    response = await ds_client.get(\"/-/static/app.css\")\n    assert response.status_code == 200\n    assert \"text/css\" == response.headers[\"content-type\"]\n    assert \"etag\" in response.headers\n    etag = response.headers.get(\"etag\")\n    response = await ds_client.get(\"/-/static/app.css\", headers={\"if-none-match\": etag})\n    assert response.status_code == 304\n\n\ndef test_static_mounts():\n    with make_app_client(\n        static_mounts=[(\"custom-static\", str(pathlib.Path(__file__).parent))]\n    ) as client:\n        response = client.get(\"/custom-static/test_html.py\")\n        assert response.status_code == 200\n        response = client.get(\"/custom-static/not_exists.py\")\n        assert response.status_code == 404\n        response = client.get(\"/custom-static/../LICENSE\")\n        assert response.status_code == 404\n\n\ndef test_memory_database_page():\n    with make_app_client(memory=True) as client:\n        response = client.get(\"/_memory\")\n        assert response.status_code == 200\n\n\ndef test_not_allowed_methods():\n    with make_app_client(memory=True) as client:\n        for method in (\"post\", \"put\", \"patch\", \"delete\"):\n            response = client.request(path=\"/_memory\", method=method.upper())\n            assert response.status_code == 405\n\n\n@pytest.mark.asyncio\nasync def test_database_page(ds_client):\n    response = await ds_client.get(\"/fixtures\")\n    soup = Soup(response.text, \"html.parser\")\n    # Should have a ',\n    ]\n    for expected_html_fragment in expected_html_fragments:\n        assert expected_html_fragment in response.text\n\n\ndef test_row_page_does_not_truncate():\n    with make_app_client(settings={\"truncate_cells_html\": 5}) as client:\n        response = client.get(\"/fixtures/facetable/1\")\n        assert response.status_code == 200\n        table = Soup(response.content, \"html.parser\").find(\"table\")\n        assert table[\"class\"] == [\"rows-and-columns\"]\n        assert [\"Mission\"] == [\n            td.string\n            for td in table.find_all(\"td\", {\"class\": \"col-neighborhood-b352a7\"})\n        ]\n\n\ndef test_query_page_truncates():\n    with make_app_client(settings={\"truncate_cells_html\": 5}) as client:\n        response = client.get(\n            \"/fixtures/-/query?\"\n            + urllib.parse.urlencode(\n                {\n                    \"sql\": \"select 'this is longer than 5' as a, 'https://example.com/' as b\"\n                }\n            )\n        )\n        assert response.status_code == 200\n        table = Soup(response.content, \"html.parser\").find(\"table\")\n        tds = table.find_all(\"td\")\n        assert [str(td) for td in tds] == [\n            '
    • ',\n            '',\n        ]\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"path,expected_classes\",\n    [\n        (\"/\", [\"index\"]),\n        (\"/fixtures\", [\"db\", \"db-fixtures\"]),\n        (\"/fixtures/-/query?sql=select+1\", [\"query\", \"db-fixtures\"]),\n        (\n            \"/fixtures/simple_primary_key\",\n            [\"table\", \"db-fixtures\", \"table-simple_primary_key\"],\n        ),\n        (\n            \"/fixtures/neighborhood_search\",\n            [\"query\", \"db-fixtures\", \"query-neighborhood_search\"],\n        ),\n        (\n            \"/fixtures/table~2Fwith~2Fslashes~2Ecsv\",\n            [\"table\", \"db-fixtures\", \"table-tablewithslashescsv-fa7563\"],\n        ),\n        (\n            \"/fixtures/simple_primary_key/1\",\n            [\"row\", \"db-fixtures\", \"table-simple_primary_key\"],\n        ),\n    ],\n)\nasync def test_css_classes_on_body(ds_client, path, expected_classes):\n    response = await ds_client.get(path)\n    assert response.status_code == 200\n    classes = re.search(r'', response.text).group(1).split()\n    assert classes == expected_classes\n\n\ntemplates_considered_re = re.compile(r\"\")\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"path,expected_considered\",\n    [\n        (\"/\", \"*index.html\"),\n        (\"/fixtures\", \"database-fixtures.html, *database.html\"),\n        (\n            \"/fixtures/simple_primary_key\",\n            \"table-fixtures-simple_primary_key.html, *table.html\",\n        ),\n        (\n            \"/fixtures/table~2Fwith~2Fslashes~2Ecsv\",\n            \"table-fixtures-tablewithslashescsv-fa7563.html, *table.html\",\n        ),\n        (\n            \"/fixtures/simple_primary_key/1\",\n            \"row-fixtures-simple_primary_key.html, *row.html\",\n        ),\n    ],\n)\nasync def test_templates_considered(ds_client, path, expected_considered):\n    response = await ds_client.get(path)\n    assert response.status_code == 200\n    match = templates_considered_re.search(response.text)\n    assert match, \"No templates considered comment found\"\n    actual_considered = match.group(1)\n    assert actual_considered == expected_considered\n\n\n@pytest.mark.asyncio\nasync def test_row_json_export_link(ds_client):\n    response = await ds_client.get(\"/fixtures/simple_primary_key/1\")\n    assert response.status_code == 200\n    assert 'json' in response.text\n\n\n@pytest.mark.asyncio\nasync def test_query_json_csv_export_links(ds_client):\n    response = await ds_client.get(\"/fixtures/-/query?sql=select+1\")\n    assert response.status_code == 200\n    assert 'json' in response.text\n    assert (\n        'CSV'\n        in response.text\n    )\n\n\n@pytest.mark.asyncio\nasync def test_query_parameter_form_fields(ds_client):\n    response = await ds_client.get(\"/fixtures/-/query?sql=select+:name\")\n    assert response.status_code == 200\n    assert (\n        ' '\n        in response.text\n    )\n    response2 = await ds_client.get(\"/fixtures/-/query?sql=select+:name&name=hello\")\n    assert response2.status_code == 200\n    assert (\n        ' '\n        in response2.text\n    )\n\n\n@pytest.mark.asyncio\nasync def test_row_html_simple_primary_key(ds_client):\n    response = await ds_client.get(\"/fixtures/simple_primary_key/1\")\n    assert response.status_code == 200\n    table = Soup(response.content, \"html.parser\").find(\"table\")\n    assert [\"id\", \"content\"] == [th.string.strip() for th in table.select(\"thead th\")]\n    assert [\n        [\n            '',\n            '',\n        ]\n    ] == [[str(td) for td in tr.select(\"td\")] for tr in table.select(\"tbody tr\")]\n\n\n@pytest.mark.asyncio\nasync def test_row_html_no_primary_key(ds_client):\n    response = await ds_client.get(\"/fixtures/no_primary_key/1\")\n    assert response.status_code == 200\n    table = Soup(response.text, \"html.parser\").find(\"table\")\n    assert [\"rowid\", \"content\", \"a\", \"b\", \"c\"] == [\n        th.string.strip() for th in table.select(\"thead th\")\n    ]\n    expected = [\n        [\n            '',\n            '',\n            '',\n            '',\n            '',\n        ]\n    ]\n    assert expected == [\n        [str(td) for td in tr.select(\"td\")] for tr in table.select(\"tbody tr\")\n    ]\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"path,expected_text,expected_link\",\n    (\n        (\n            \"/fixtures/facet_cities/1\",\n            \"6 rows from _city_id in facetable\",\n            \"/fixtures/facetable?_city_id__exact=1\",\n        ),\n        (\n            \"/fixtures/attraction_characteristic/2\",\n            \"3 rows from characteristic_id in roadside_attraction_characteristics\",\n            \"/fixtures/roadside_attraction_characteristics?characteristic_id=2\",\n        ),\n    ),\n)\nasync def test_row_links_from_other_tables(\n    ds_client, path, expected_text, expected_link\n):\n    response = await ds_client.get(path)\n    assert response.status_code == 200\n    soup = Soup(response.text, \"html.parser\")\n    h2 = soup.find(\"h2\")\n    assert h2.text == \"Links from other tables\"\n    li = h2.find_next(\"ul\").find(\"li\")\n    text = re.sub(r\"\\s+\", \" \", li.text.strip())\n    assert text == expected_text\n    link = li.find(\"a\")[\"href\"]\n    assert link == expected_link\n\n\n@pytest.mark.asyncio\nasync def test_row_foreign_key_links(ds_client):\n    # Row detail page should render foreign key values as hyperlinks\n    response = await ds_client.get(\"/fixtures/foreign_key_references/1\")\n    assert response.status_code == 200\n    soup = Soup(response.text, \"html.parser\")\n    # foreign_key_with_label=1 references simple_primary_key(id=1, content=\"hello\")\n    td = soup.find(\"td\", {\"class\": \"col-foreign_key_with_label\"})\n    a = td.find(\"a\")\n    assert a is not None, \"Expected foreign key value to be a hyperlink\"\n    assert a[\"href\"] == \"/fixtures/simple_primary_key/1\"\n    assert a.text == \"hello\"\n    # Primary key column should be first and bold\n    table = soup.find(\"table\")\n    headers = [th.text.strip() for th in table.select(\"thead th\")]\n    assert headers[0] == \"pk\"\n    first_td = table.select(\"tbody tr td\")[0]\n    assert first_td.find(\"strong\") is not None, \"PK value should be bold\"\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"path,expected\",\n    (\n        (\n            \"/fixtures/compound_primary_key/a,b\",\n            [\n                [\n                    '',\n                    '',\n                    '',\n                ]\n            ],\n        ),\n        (\n            \"/fixtures/compound_primary_key/a~2Fb,~2Ec~2Dd\",\n            [\n                [\n                    '',\n                    '',\n                    '',\n                ]\n            ],\n        ),\n    ),\n)\nasync def test_row_html_compound_primary_key(ds_client, path, expected):\n    response = await ds_client.get(path)\n    assert response.status_code == 200\n    table = Soup(response.text, \"html.parser\").find(\"table\")\n    assert [\"pk1\", \"pk2\", \"content\"] == [\n        th.string.strip() for th in table.select(\"thead th\")\n    ]\n    assert expected == [\n        [str(td) for td in tr.select(\"td\")] for tr in table.select(\"tbody tr\")\n    ]\n\n\n@pytest.mark.asyncio\nasync def test_index_metadata(ds_client):\n    response = await ds_client.get(\"/\")\n    assert response.status_code == 200\n    soup = Soup(response.text, \"html.parser\")\n    assert \"Datasette Fixtures\" == soup.find(\"h1\").text\n    assert (\n        'An example SQLite database demonstrating Datasette. Sign in as root user'\n        == inner_html(soup.find(\"div\", {\"class\": \"metadata-description\"}))\n    )\n    assert_footer_links(soup)\n\n\n@pytest.mark.asyncio\nasync def test_database_metadata(ds_client):\n    response = await ds_client.get(\"/fixtures\")\n    assert response.status_code == 200\n    soup = Soup(response.text, \"html.parser\")\n    # Page title should be the default\n    assert \"fixtures\" == soup.find(\"h1\").text\n    # Description should be custom\n    assert \"Test tables description\" == inner_html(\n        soup.find(\"div\", {\"class\": \"metadata-description\"})\n    )\n    # The source/license should be inherited\n    # assert_footer_links(soup) TODO(alex) ensure\n\n\n@pytest.mark.asyncio\nasync def test_database_metadata_with_custom_sql(ds_client):\n    response = await ds_client.get(\n        \"/fixtures/-/query?sql=select+*+from+simple_primary_key\"\n    )\n    assert response.status_code == 200\n    soup = Soup(response.text, \"html.parser\")\n    # Page title should be the default\n    assert \"fixtures\" == soup.find(\"h1\").text\n    # Description should be custom\n    assert \"Custom SQL query returning\" in soup.find(\"h3\").text\n    # The source/license should be inherited\n    # assert_footer_links(soup)TODO(alex) ensure\n\n\ndef test_database_download_for_immutable():\n    with make_app_client(is_immutable=True) as client:\n        assert not client.ds.databases[\"fixtures\"].is_mutable\n        # Regular page should have a download link\n        response = client.get(\"/fixtures\")\n        soup = Soup(response.content, \"html.parser\")\n        assert len(soup.find_all(\"a\", {\"href\": re.compile(r\"\\.db$\")}))\n        # Check we can actually download it\n        download_response = client.get(\"/fixtures.db\")\n        assert download_response.status_code == 200\n        # Check the content-length header exists\n        assert \"content-length\" in download_response.headers\n        content_length = download_response.headers[\"content-length\"]\n        assert content_length.isdigit()\n        assert int(content_length) > 100\n        assert \"content-disposition\" in download_response.headers\n        assert (\n            download_response.headers[\"content-disposition\"]\n            == 'attachment; filename=\"fixtures.db\"'\n        )\n        assert download_response.headers[\"transfer-encoding\"] == \"chunked\"\n        # ETag header should be present and match db.hash\n        assert \"etag\" in download_response.headers\n        etag = download_response.headers[\"etag\"]\n        assert etag == '\"{}\"'.format(client.ds.databases[\"fixtures\"].hash)\n        # Try a second download with If-None-Match: current-etag\n        download_response2 = client.get(\"/fixtures.db\", if_none_match=etag)\n        assert download_response2.body == b\"\"\n        assert download_response2.status == 304\n\n\ndef test_database_download_disallowed_for_mutable(app_client):\n    # Use app_client because we need a file database, not in-memory\n    response = app_client.get(\"/fixtures\")\n    soup = Soup(response.content, \"html.parser\")\n    assert len(soup.find_all(\"a\", {\"href\": re.compile(r\"\\.db$\")})) == 0\n    assert app_client.get(\"/fixtures.db\").status_code == 403\n\n\ndef test_database_download_disallowed_for_memory():\n    with make_app_client(memory=True) as client:\n        # Memory page should NOT have a download link\n        response = client.get(\"/_memory\")\n        soup = Soup(response.content, \"html.parser\")\n        assert 0 == len(soup.find_all(\"a\", {\"href\": re.compile(r\"\\.db$\")}))\n        assert 404 == client.get(\"/_memory.db\").status\n\n\ndef test_allow_download_off():\n    with make_app_client(\n        is_immutable=True, settings={\"allow_download\": False}\n    ) as client:\n        response = client.get(\"/fixtures\")\n        soup = Soup(response.content, \"html.parser\")\n        assert not len(soup.find_all(\"a\", {\"href\": re.compile(r\"\\.db$\")}))\n        # Accessing URL directly should 403\n        response = client.get(\"/fixtures.db\")\n        assert 403 == response.status\n\n\ndef test_allow_sql_off():\n    with make_app_client(config={\"allow_sql\": {}}) as client:\n        response = client.get(\"/fixtures\")\n        soup = Soup(response.content, \"html.parser\")\n        assert not len(soup.find_all(\"textarea\", {\"name\": \"sql\"}))\n        # The table page should no longer show \"View and edit SQL\"\n        response = client.get(\"/fixtures/sortable\")\n        assert b\"View and edit SQL\" not in response.content\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\"path\", [\"/404\", \"/fixtures/404\"])\nasync def test_404(ds_client, path):\n    response = await ds_client.get(path)\n    assert response.status_code == 404\n    assert (\n        f'\n \n  Demonstrating\n \n simple like search\n\"\"\".strip()\n        == soup.find(\"div\", {\"class\": \"metadata-description\"}).prettify().strip()\n    )\n\n\n@pytest.mark.asyncio\nasync def test_urlify_custom_queries(ds_client):\n    path = \"/fixtures/-/query?\" + urllib.parse.urlencode(\n        {\"sql\": \"select ('https://twitter.com/' || 'simonw') as user_url;\"}\n    )\n    response = await ds_client.get(path)\n    assert response.status_code == 200\n    soup = Soup(response.content, \"html.parser\")\n    assert \"\"\"\"\"\" == soup.find(\"td\", {\"class\": \"col-user_url\"}).prettify().strip()\n\n\n@pytest.mark.asyncio\nasync def test_show_hide_sql_query(ds_client):\n    path = \"/fixtures/-/query?\" + urllib.parse.urlencode(\n        {\"sql\": \"select ('https://twitter.com/' || 'simonw') as user_url;\"}\n    )\n    response = await ds_client.get(path)\n    soup = Soup(response.content, \"html.parser\")\n    span = soup.select(\".show-hide-sql\")[0]\n    assert span.find(\"a\")[\"href\"].endswith(\"&_hide_sql=1\")\n    assert \"(hide)\" == span.getText()\n    assert soup.find(\"textarea\") is not None\n    # Now follow the link to hide it\n    response = await ds_client.get(span.find(\"a\")[\"href\"])\n    soup = Soup(response.content, \"html.parser\")\n    span = soup.select(\".show-hide-sql\")[0]\n    assert not span.find(\"a\")[\"href\"].endswith(\"&_hide_sql=1\")\n    assert \"(show)\" == span.getText()\n    assert soup.find(\"textarea\") is None\n    # The SQL should still be there in a hidden form field\n    hiddens = soup.find(\"form\").select(\"input[type=hidden]\")\n    assert [\n        (\"sql\", \"select ('https://twitter.com/' || 'simonw') as user_url;\"),\n        (\"_hide_sql\", \"1\"),\n    ] == [(hidden[\"name\"], hidden[\"value\"]) for hidden in hiddens]\n\n\n@pytest.mark.asyncio\nasync def test_canned_query_with_hide_has_no_hidden_sql(ds_client):\n    # For a canned query the show/hide should NOT have a hidden SQL field\n    # https://github.com/simonw/datasette/issues/1411\n    response = await ds_client.get(\"/fixtures/pragma_cache_size?_hide_sql=1\")\n    soup = Soup(response.content, \"html.parser\")\n    hiddens = soup.find(\"form\").select(\"input[type=hidden]\")\n    assert [\n        (\"_hide_sql\", \"1\"),\n    ] == [(hidden[\"name\"], hidden[\"value\"]) for hidden in hiddens]\n\n\n@pytest.mark.parametrize(\n    \"hide_sql,querystring,expected_hidden,expected_show_hide_link,expected_show_hide_text\",\n    (\n        (False, \"\", None, \"/_memory/one?_hide_sql=1\", \"hide\"),\n        (False, \"?_hide_sql=1\", \"_hide_sql\", \"/_memory/one\", \"show\"),\n        (True, \"\", None, \"/_memory/one?_show_sql=1\", \"show\"),\n        (True, \"?_show_sql=1\", \"_show_sql\", \"/_memory/one\", \"hide\"),\n    ),\n)\ndef test_canned_query_show_hide_metadata_option(\n    hide_sql,\n    querystring,\n    expected_hidden,\n    expected_show_hide_link,\n    expected_show_hide_text,\n):\n    with make_app_client(\n        config={\n            \"databases\": {\n                \"_memory\": {\n                    \"queries\": {\n                        \"one\": {\n                            \"sql\": \"select 1 + 1\",\n                            \"hide_sql\": hide_sql,\n                        }\n                    }\n                }\n            }\n        },\n        memory=True,\n    ) as client:\n        expected_show_hide_fragment = '({})'.format(\n            expected_show_hide_link, expected_show_hide_text\n        )\n        response = client.get(\"/_memory/one\" + querystring)\n        html = response.text\n        show_hide_fragment = html.split('')[1].split(\n            \"\"\n        )[0]\n        assert show_hide_fragment == expected_show_hide_fragment\n        if expected_hidden:\n            assert (\n                ''.format(expected_hidden)\n                in html\n            )\n        else:\n            assert '<Binary:\\xa07\\xa0bytes>'\n        ],\n        [\n            ''\n        ],\n        [''],\n    ]\n    assert expected_tds == [\n        [str(td) for td in tr.select(\"td\")] for tr in table.select(\"tbody tr\")\n    ]\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"path,expected_filename\",\n    [\n        (\"/fixtures/binary_data/1.blob?_blob_column=data\", \"binary_data-1-data.blob\"),\n        (\n            \"/fixtures/-/query.blob?sql=select+*+from+binary_data&_blob_column=data&_blob_hash=f3088978da8f9aea479ffc7f631370b968d2e855eeb172bea7f6c7a04262bb6d\",\n            \"data-f30889.blob\",\n        ),\n    ],\n)\nasync def test_blob_download(ds_client, path, expected_filename):\n    response = await ds_client.get(path)\n    assert response.status_code == 200\n    assert response.content == b\"\\x15\\x1c\\x02\\xc7\\xad\\x05\\xfe\"\n    assert response.headers[\"x-content-type-options\"] == \"nosniff\"\n    assert (\n        response.headers[\"content-disposition\"]\n        == f'attachment; filename=\"{expected_filename}\"'\n    )\n    assert response.headers[\"content-type\"] == \"application/binary\"\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"path,expected_message\",\n    [\n        (\"/fixtures/binary_data/1.blob\", \"?_blob_column= is required\"),\n        (\"/fixtures/binary_data/1.blob?_blob_column=foo\", \"foo is not a valid column\"),\n        (\n            \"/fixtures/binary_data/1.blob?_blob_column=data&_blob_hash=x\",\n            \"Link has expired - the requested binary content has changed or could not be found.\",\n        ),\n    ],\n)\nasync def test_blob_download_invalid_messages(ds_client, path, expected_message):\n    response = await ds_client.get(path)\n    assert response.status_code == 400\n    assert expected_message in response.text\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"path\",\n    [\n        \"/fixtures/-/query?sql=select+*+from+[123_starts_with_digits]\",\n        \"/fixtures/123_starts_with_digits\",\n    ],\n)\nasync def test_zero_results(ds_client, path):\n    response = await ds_client.get(path)\n    soup = Soup(response.text, \"html.parser\")\n    assert 0 == len(soup.select(\"table\"))\n    assert 1 == len(soup.select(\"p.zero-results\"))\n\n\n@pytest.mark.asyncio\nasync def test_query_error(ds_client):\n    response = await ds_client.get(\"/fixtures/-/query?sql=select+*+from+notatable\")\n    html = response.text\n    assert '
    no such table: notatable
    ' in html\n    assert '\" in html\n    assert \"0 results\" not in html\n\n\ndef test_config_template_debug_on():\n    with make_app_client(settings={\"template_debug\": True}) as client:\n        response = client.get(\"/fixtures/facetable?_context=1\")\n        assert response.status_code == 200\n        assert response.text.startswith(\"
    {\")\n\n\n@pytest.mark.asyncio\nasync def test_config_template_debug_off(ds_client):\n    response = await ds_client.get(\"/fixtures/facetable?_context=1\")\n    assert response.status_code == 200\n    assert not response.text.startswith(\"
    {\")\n\n\ndef test_debug_context_includes_extra_template_vars():\n    # https://github.com/simonw/datasette/issues/693\n    with make_app_client(settings={\"template_debug\": True}) as client:\n        response = client.get(\"/fixtures/facetable?_context=1\")\n        # scope_path is added by PLUGIN1\n        assert \"scope_path\" in response.text\n\n\n@pytest.mark.parametrize(\n    \"path\",\n    [\n        \"/\",\n        \"/fixtures\",\n        \"/fixtures/compound_three_primary_keys\",\n        \"/fixtures/compound_three_primary_keys/a,a,a\",\n        \"/fixtures/paginated_view\",\n        \"/fixtures/facetable\",\n        \"/fixtures/facetable?_facet=state\",\n        \"/fixtures/-/query?sql=select+1\",\n    ],\n)\n@pytest.mark.parametrize(\"use_prefix\", (True, False))\ndef test_base_url_config(app_client_base_url_prefix, path, use_prefix):\n    client = app_client_base_url_prefix\n    path_to_get = path\n    if use_prefix:\n        path_to_get = \"/prefix/\" + path.lstrip(\"/\")\n    response = client.get(path_to_get)\n    soup = Soup(response.content, \"html.parser\")\n    for form in soup.select(\"form\"):\n        action = form.get(\"action\")\n        if action is None:\n            assert form.get(\"method\") == \"dialog\", json.dumps(\n                {\n                    \"path\": path,\n                    \"path_to_get\": path_to_get,\n                    \"form\": str(form),\n                },\n                indent=4,\n                default=repr,\n            )\n            continue\n        assert action.startswith(\"/prefix\"), json.dumps(\n            {\n                \"path\": path,\n                \"path_to_get\": path_to_get,\n                \"action\": action,\n                \"form\": str(form),\n            },\n            indent=4,\n            default=repr,\n        )\n    for el in soup.find_all([\"a\", \"link\", \"script\"]):\n        if \"href\" in el.attrs:\n            href = el[\"href\"]\n        elif \"src\" in el.attrs:\n            href = el[\"src\"]\n        else:\n            continue  # Could be a \n        if (\n            not href.startswith(\"#\")\n            and href\n            not in {\n                \"https://datasette.io/\",\n                \"https://github.com/simonw/datasette\",\n                \"https://github.com/simonw/datasette/blob/main/LICENSE\",\n                \"https://github.com/simonw/datasette/blob/main/tests/fixtures.py\",\n                \"/login-as-root\",  # Only used for the latest.datasette.io demo\n            }\n            and not href.startswith(\"https://plugin-example.datasette.io/\")\n        ):\n            # If this has been made absolute it may start http://localhost/\n            if href.startswith(\"http://localhost/\"):\n                href = href[len(\"http://localost/\") :]\n            assert href.startswith(\"/prefix/\"), json.dumps(\n                {\n                    \"path\": path,\n                    \"path_to_get\": path_to_get,\n                    \"href_or_src\": href,\n                    \"element_parent\": str(el.parent),\n                },\n                indent=4,\n                default=repr,\n            )\n\n\ndef test_base_url_affects_filter_redirects(app_client_base_url_prefix):\n    path = \"/fixtures/binary_data?_filter_column=rowid&_filter_op=exact&_filter_value=1&_sort=rowid\"\n    response = app_client_base_url_prefix.get(path)\n    assert response.status_code == 302\n    assert (\n        response.headers[\"location\"]\n        == \"/prefix/fixtures/binary_data?_sort=rowid&rowid__exact=1\"\n    )\n\n\ndef test_base_url_affects_metadata_extra_css_urls(app_client_base_url_prefix):\n    html = app_client_base_url_prefix.get(\"/\").text\n    assert '' in html\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"path,expected\",\n    [\n        (\n            \"/fixtures/neighborhood_search\",\n            \"/fixtures/-/query?sql=%0Aselect+_neighborhood%2C+facet_cities.name%2C+state%0Afrom+facetable%0A++++join+facet_cities%0A++++++++on+facetable._city_id+%3D+facet_cities.id%0Awhere+_neighborhood+like+%27%25%27+%7C%7C+%3Atext+%7C%7C+%27%25%27%0Aorder+by+_neighborhood%3B%0A&text=\",\n        ),\n        (\n            \"/fixtures/neighborhood_search?text=ber\",\n            \"/fixtures/-/query?sql=%0Aselect+_neighborhood%2C+facet_cities.name%2C+state%0Afrom+facetable%0A++++join+facet_cities%0A++++++++on+facetable._city_id+%3D+facet_cities.id%0Awhere+_neighborhood+like+%27%25%27+%7C%7C+%3Atext+%7C%7C+%27%25%27%0Aorder+by+_neighborhood%3B%0A&text=ber\",\n        ),\n        (\"/fixtures/pragma_cache_size\", None),\n        (\n            # /fixtures/𝐜𝐢𝐭𝐢𝐞𝐬\n            \"/fixtures/~F0~9D~90~9C~F0~9D~90~A2~F0~9D~90~AD~F0~9D~90~A2~F0~9D~90~9E~F0~9D~90~AC\",\n            \"/fixtures/-/query?sql=select+id%2C+name+from+facet_cities+order+by+id+limit+1%3B\",\n        ),\n        (\"/fixtures/magic_parameters\", None),\n    ],\n)\nasync def test_edit_sql_link_on_canned_queries(ds_client, path, expected):\n    response = await ds_client.get(path)\n    assert response.status_code == 200\n    expected_link = f'Edit SQL'\n    if expected:\n        assert expected_link in response.text\n    else:\n        assert \"Edit SQL\" not in response.text\n\n\n@pytest.mark.parametrize(\n    \"has_permission\",\n    [\n        pytest.param(\n            True,\n        ),\n        False,\n    ],\n)\ndef test_edit_sql_link_not_shown_if_user_lacks_permission(has_permission):\n    with make_app_client(\n        config={\n            \"allow_sql\": None if has_permission else {\"id\": \"not-you\"},\n            \"databases\": {\"fixtures\": {\"queries\": {\"simple\": \"select 1 + 1\"}}},\n        }\n    ) as client:\n        response = client.get(\"/fixtures/simple\")\n        if has_permission:\n            assert \"Edit SQL\" in response.text\n        else:\n            assert \"Edit SQL\" not in response.text\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"actor_id,should_have_links,should_not_have_links\",\n    [\n        (None, None, None),\n        (\"test\", None, [\"/-/permissions\"]),\n        (\"root\", [\"/-/permissions\", \"/-/allow-debug\"], None),\n    ],\n)\nasync def test_navigation_menu_links(\n    ds_client, actor_id, should_have_links, should_not_have_links\n):\n    # Enable root user if testing with root actor\n    if actor_id == \"root\":\n        ds_client.ds.root_enabled = True\n    cookies = {}\n    if actor_id:\n        cookies = {\"ds_actor\": ds_client.actor_cookie({\"id\": actor_id})}\n    html = (await ds_client.get(\"/\", cookies=cookies)).text\n    soup = Soup(html, \"html.parser\")\n    details = soup.find(\"nav\").find(\"details\")\n    if not actor_id:\n        # Should not show a menu\n        assert details is None\n        return\n    # They are logged in: should show a menu\n    assert details is not None\n    # And a logout form\n    assert details.find(\"form\") is not None\n    if should_have_links:\n        for link in should_have_links:\n            assert (\n                details.find(\"a\", {\"href\": link}) is not None\n            ), f\"{link} expected but missing from nav menu\"\n\n    if should_not_have_links:\n        for link in should_not_have_links:\n            assert (\n                details.find(\"a\", {\"href\": link}) is None\n            ), f\"{link} found but should not have been in nav menu\"\n\n\n@pytest.mark.asyncio\nasync def test_trace_correctly_escaped(ds_client):\n    response = await ds_client.get(\"/fixtures/-/query?sql=select+'
    Hello'&_trace=1\")\n    assert \"select '
    Hello\" not in response.text\n    assert \"select '<h1>Hello\" in response.text\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"path,expected\",\n    (\n        # Instance index page\n        (\"/\", \"http://localhost/.json\"),\n        # Table page\n        (\"/fixtures/facetable\", \"http://localhost/fixtures/facetable.json\"),\n        (\n            \"/fixtures/table~2Fwith~2Fslashes~2Ecsv\",\n            \"http://localhost/fixtures/table~2Fwith~2Fslashes~2Ecsv.json\",\n        ),\n        # Row page\n        (\n            \"/fixtures/no_primary_key/1\",\n            \"http://localhost/fixtures/no_primary_key/1.json\",\n        ),\n        # Database index page\n        (\n            \"/fixtures\",\n            \"http://localhost/fixtures.json\",\n        ),\n        # Custom query page\n        (\n            \"/fixtures/-/query?sql=select+*+from+facetable\",\n            \"http://localhost/fixtures/-/query.json?sql=select+*+from+facetable\",\n        ),\n        # Canned query page\n        (\n            \"/fixtures/neighborhood_search?text=town\",\n            \"http://localhost/fixtures/neighborhood_search.json?text=town\",\n        ),\n        # /-/ pages\n        (\n            \"/-/plugins\",\n            \"http://localhost/-/plugins.json\",\n        ),\n    ),\n)\nasync def test_alternate_url_json(ds_client, path, expected):\n    response = await ds_client.get(path)\n    assert response.status_code == 200\n    link = response.headers[\"link\"]\n    assert link == '<{}>; rel=\"alternate\"; type=\"application/json+datasette\"'.format(\n        expected\n    )\n    assert (\n        ''.format(\n            expected\n        )\n        in response.text\n    )\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"path\",\n    (\"/-/patterns\", \"/-/messages\", \"/-/allow-debug\", \"/fixtures.db\"),\n)\nasync def test_no_alternate_url_json(ds_client, path):\n    response = await ds_client.get(path)\n    assert \"link\" not in response.headers\n    assert (\n        'Name\" in response.text\n        assert \"view-instance\" in response.text\n        assert \"view-database\" in response.text\n    finally:\n        ds_client.ds.root_enabled = original_root_enabled\n\n\n@pytest.mark.asyncio\nasync def test_actions_page_does_not_display_none_string(ds_client):\n    \"\"\"Ensure the Resource column doesn't display the string 'None' for null values.\"\"\"\n    # https://github.com/simonw/datasette/issues/2599\n    original_root_enabled = ds_client.ds.root_enabled\n    try:\n        ds_client.ds.root_enabled = True\n        cookies = {\"ds_actor\": ds_client.actor_cookie({\"id\": \"root\"})}\n        response = await ds_client.get(\"/-/actions\", cookies=cookies)\n        assert response.status_code == 200\n        assert \"None\" not in response.text\n    finally:\n        ds_client.ds.root_enabled = original_root_enabled\n\n\n@pytest.mark.asyncio\nasync def test_permission_debug_tabs_with_query_string(ds_client):\n    \"\"\"Test that navigation tabs persist query strings across Check, Allowed, and Rules pages\"\"\"\n    original_root_enabled = ds_client.ds.root_enabled\n    try:\n        ds_client.ds.root_enabled = True\n        cookies = {\"ds_actor\": ds_client.actor_cookie({\"id\": \"root\"})}\n\n        # Test /-/allowed with query string\n        response = await ds_client.get(\n            \"/-/allowed?action=view-table&page_size=50\", cookies=cookies\n        )\n        assert response.status_code == 200\n        # Check that Rules and Check tabs have the query string\n        assert 'href=\"/-/rules?action=view-table&page_size=50\"' in response.text\n        assert 'href=\"/-/check?action=view-table&page_size=50\"' in response.text\n        # Playground and Actions should not have query string\n        assert 'href=\"/-/permissions\"' in response.text\n        assert 'href=\"/-/actions\"' in response.text\n\n        # Test /-/rules with query string\n        response = await ds_client.get(\n            \"/-/rules?action=view-database&parent=test\", cookies=cookies\n        )\n        assert response.status_code == 200\n        # Check that Allowed and Check tabs have the query string\n        assert 'href=\"/-/allowed?action=view-database&parent=test\"' in response.text\n        assert 'href=\"/-/check?action=view-database&parent=test\"' in response.text\n\n        # Test /-/check with query string\n        response = await ds_client.get(\"/-/check?action=execute-sql\", cookies=cookies)\n        assert response.status_code == 200\n        # Check that Allowed and Rules tabs have the query string\n        assert 'href=\"/-/allowed?action=execute-sql\"' in response.text\n        assert 'href=\"/-/rules?action=execute-sql\"' in response.text\n    finally:\n        ds_client.ds.root_enabled = original_root_enabled\n"
  },
  {
    "path": "tests/test_internal_db.py",
    "content": "import pytest\nimport sqlite_utils\n\n\n# ensure refresh_schemas() gets called before interacting with internal_db\nasync def ensure_internal(ds_client):\n    await ds_client.get(\"/fixtures.json?sql=select+1\")\n    return ds_client.ds.get_internal_database()\n\n\n@pytest.mark.asyncio\nasync def test_internal_databases(ds_client):\n    internal_db = await ensure_internal(ds_client)\n    databases = await internal_db.execute(\"select * from catalog_databases\")\n    assert len(databases) == 1\n    assert databases.rows[0][\"database_name\"] == \"fixtures\"\n\n\n@pytest.mark.asyncio\nasync def test_internal_tables(ds_client):\n    internal_db = await ensure_internal(ds_client)\n    tables = await internal_db.execute(\"select * from catalog_tables\")\n    assert len(tables) > 5\n    table = tables.rows[0]\n    assert set(table.keys()) == {\"rootpage\", \"table_name\", \"database_name\", \"sql\"}\n\n\n@pytest.mark.asyncio\nasync def test_internal_views(ds_client):\n    internal_db = await ensure_internal(ds_client)\n    views = await internal_db.execute(\"select * from catalog_views\")\n    assert len(views) >= 4\n    view = views.rows[0]\n    assert set(view.keys()) == {\"rootpage\", \"view_name\", \"database_name\", \"sql\"}\n\n\n@pytest.mark.asyncio\nasync def test_internal_indexes(ds_client):\n    internal_db = await ensure_internal(ds_client)\n    indexes = await internal_db.execute(\"select * from catalog_indexes\")\n    assert len(indexes) > 5\n    index = indexes.rows[0]\n    assert set(index.keys()) == {\n        \"partial\",\n        \"name\",\n        \"table_name\",\n        \"unique\",\n        \"seq\",\n        \"database_name\",\n        \"origin\",\n    }\n\n\n@pytest.mark.asyncio\nasync def test_internal_foreign_keys(ds_client):\n    internal_db = await ensure_internal(ds_client)\n    foreign_keys = await internal_db.execute(\"select * from catalog_foreign_keys\")\n    assert len(foreign_keys) > 5\n    foreign_key = foreign_keys.rows[0]\n    assert set(foreign_key.keys()) == {\n        \"table\",\n        \"seq\",\n        \"on_update\",\n        \"on_delete\",\n        \"to\",\n        \"id\",\n        \"match\",\n        \"database_name\",\n        \"table_name\",\n        \"from\",\n    }\n\n\n@pytest.mark.asyncio\nasync def test_internal_foreign_key_references(ds_client):\n    internal_db = await ensure_internal(ds_client)\n\n    def inner(conn):\n        db = sqlite_utils.Database(conn)\n        table_names = db.table_names()\n        for table in db.tables:\n            for fk in table.foreign_keys:\n                other_table = fk.other_table\n                other_column = fk.other_column\n                message = 'Column \"{}.{}\" references other column \"{}.{}\" which does not exist'.format(\n                    table.name, fk.column, other_table, other_column\n                )\n                assert other_table in table_names, message + \" (bad table)\"\n                assert other_column in db[other_table].columns_dict, (\n                    message + \" (bad column)\"\n                )\n\n    await internal_db.execute_fn(inner)\n\n\n@pytest.mark.asyncio\nasync def test_stale_catalog_entry_database_fix(tmp_path):\n    \"\"\"\n    Test for https://github.com/simonw/datasette/issues/2605\n\n    When the internal database persists across restarts and has entries in\n    catalog_databases for databases that no longer exist, accessing the\n    index page should not cause a 500 error (KeyError).\n    \"\"\"\n    from datasette.app import Datasette\n\n    internal_db_path = str(tmp_path / \"internal.db\")\n    data_db_path = str(tmp_path / \"data.db\")\n\n    # Create a data database file\n    import sqlite3\n\n    conn = sqlite3.connect(data_db_path)\n    conn.execute(\"CREATE TABLE test_table (id INTEGER PRIMARY KEY)\")\n    conn.close()\n\n    # First Datasette instance: with the data database and persistent internal db\n    ds1 = Datasette(files=[data_db_path], internal=internal_db_path)\n    await ds1.invoke_startup()\n\n    # Access the index page to populate the internal catalog\n    response = await ds1.client.get(\"/\")\n    assert \"data\" in ds1.databases\n    assert response.status_code == 200\n\n    # Second Datasette instance: reusing internal.db but WITHOUT the data database\n    # This simulates restarting Datasette after removing a database\n    ds2 = Datasette(internal=internal_db_path)\n    await ds2.invoke_startup()\n\n    # The database is not in ds2.databases\n    assert \"data\" not in ds2.databases\n\n    # Accessing the index page should NOT cause a 500 error\n    # This is the bug: it currently raises KeyError when trying to\n    # access ds.databases[\"data\"] for the stale catalog entry\n    response = await ds2.client.get(\"/\")\n    assert response.status_code == 200, (\n        f\"Index page should return 200, not {response.status_code}. \"\n        \"This fails due to stale catalog entries causing KeyError.\"\n    )\n"
  },
  {
    "path": "tests/test_internals_database.py",
    "content": "\"\"\"\nTests for the datasette.database.Database class\n\"\"\"\n\nfrom datasette.app import Datasette\nfrom datasette.database import Database, Results, MultipleValues\nfrom datasette.utils.sqlite import sqlite3, sqlite_version\nfrom datasette.utils import Column\nimport pytest\nimport time\nimport uuid\n\n\n@pytest.fixture\ndef db(app_client):\n    return app_client.ds.get_database(\"fixtures\")\n\n\n@pytest.mark.asyncio\nasync def test_execute(db):\n    results = await db.execute(\"select * from facetable\")\n    assert isinstance(results, Results)\n    assert 15 == len(results)\n\n\n@pytest.mark.asyncio\nasync def test_results_first(db):\n    assert None is (await db.execute(\"select * from facetable where pk > 100\")).first()\n    results = await db.execute(\"select * from facetable\")\n    row = results.first()\n    assert isinstance(row, sqlite3.Row)\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\"expected\", (True, False))\nasync def test_results_bool(db, expected):\n    where = \"\" if expected else \"where pk = 0\"\n    results = await db.execute(\"select * from facetable {}\".format(where))\n    assert bool(results) is expected\n\n\n@pytest.mark.asyncio\nasync def test_results_dicts(db):\n    results = await db.execute(\"select pk, name from roadside_attractions\")\n    assert results.dicts() == [\n        {\"pk\": 1, \"name\": \"The Mystery Spot\"},\n        {\"pk\": 2, \"name\": \"Winchester Mystery House\"},\n        {\"pk\": 3, \"name\": \"Burlingame Museum of PEZ Memorabilia\"},\n        {\"pk\": 4, \"name\": \"Bigfoot Discovery Museum\"},\n    ]\n\n\n@pytest.mark.parametrize(\n    \"query,expected\",\n    [\n        (\"select 1\", 1),\n        (\"select 1, 2\", None),\n        (\"select 1 as num union select 2 as num\", None),\n    ],\n)\n@pytest.mark.asyncio\nasync def test_results_single_value(db, query, expected):\n    results = await db.execute(query)\n    if expected:\n        assert expected == results.single_value()\n    else:\n        with pytest.raises(MultipleValues):\n            results.single_value()\n\n\n@pytest.mark.asyncio\nasync def test_execute_fn(db):\n    def get_1_plus_1(conn):\n        return conn.execute(\"select 1 + 1\").fetchall()[0][0]\n\n    assert 2 == await db.execute_fn(get_1_plus_1)\n\n\n@pytest.mark.asyncio\nasync def test_execute_fn_transaction_false():\n    datasette = Datasette(memory=True)\n    db = datasette.add_memory_database(\"test_execute_fn_transaction_false\")\n\n    def run(conn):\n        try:\n            with conn:\n                conn.execute(\"create table foo (id integer primary key)\")\n                conn.execute(\"insert into foo (id) values (44)\")\n                # Table should exist\n                assert (\n                    conn.execute(\n                        'select count(*) from sqlite_master where name = \"foo\"'\n                    ).fetchone()[0]\n                    == 1\n                )\n                assert conn.execute(\"select id from foo\").fetchall()[0][0] == 44\n                raise ValueError(\"Cancel commit\")\n        except ValueError:\n            pass\n        # Row should NOT exist\n        assert conn.execute(\"select count(*) from foo\").fetchone()[0] == 0\n\n    await db.execute_write_fn(run, transaction=False)\n\n\n@pytest.mark.parametrize(\n    \"tables,exists\",\n    (\n        ([\"facetable\", \"searchable\", \"tags\", \"searchable_tags\"], True),\n        ([\"foo\", \"bar\", \"baz\"], False),\n    ),\n)\n@pytest.mark.asyncio\nasync def test_table_exists(db, tables, exists):\n    for table in tables:\n        actual = await db.table_exists(table)\n        assert exists == actual\n\n\n@pytest.mark.parametrize(\n    \"view,expected\",\n    (\n        (\"not_a_view\", False),\n        (\"paginated_view\", True),\n    ),\n)\n@pytest.mark.asyncio\nasync def test_view_exists(db, view, expected):\n    actual = await db.view_exists(view)\n    assert actual == expected\n\n\n@pytest.mark.parametrize(\n    \"table,expected\",\n    (\n        (\n            \"facetable\",\n            [\n                \"pk\",\n                \"created\",\n                \"planet_int\",\n                \"on_earth\",\n                \"state\",\n                \"_city_id\",\n                \"_neighborhood\",\n                \"tags\",\n                \"complex_array\",\n                \"distinct_some_null\",\n                \"n\",\n            ],\n        ),\n        (\n            \"sortable\",\n            [\n                \"pk1\",\n                \"pk2\",\n                \"content\",\n                \"sortable\",\n                \"sortable_with_nulls\",\n                \"sortable_with_nulls_2\",\n                \"text\",\n            ],\n        ),\n    ),\n)\n@pytest.mark.asyncio\nasync def test_table_columns(db, table, expected):\n    columns = await db.table_columns(table)\n    assert columns == expected\n\n\n@pytest.mark.parametrize(\n    \"table,expected\",\n    (\n        (\n            \"facetable\",\n            [\n                Column(\n                    cid=0,\n                    name=\"pk\",\n                    type=\"integer\",\n                    notnull=0,\n                    default_value=None,\n                    is_pk=1,\n                    hidden=0,\n                ),\n                Column(\n                    cid=1,\n                    name=\"created\",\n                    type=\"text\",\n                    notnull=0,\n                    default_value=None,\n                    is_pk=0,\n                    hidden=0,\n                ),\n                Column(\n                    cid=2,\n                    name=\"planet_int\",\n                    type=\"integer\",\n                    notnull=0,\n                    default_value=None,\n                    is_pk=0,\n                    hidden=0,\n                ),\n                Column(\n                    cid=3,\n                    name=\"on_earth\",\n                    type=\"integer\",\n                    notnull=0,\n                    default_value=None,\n                    is_pk=0,\n                    hidden=0,\n                ),\n                Column(\n                    cid=4,\n                    name=\"state\",\n                    type=\"text\",\n                    notnull=0,\n                    default_value=None,\n                    is_pk=0,\n                    hidden=0,\n                ),\n                Column(\n                    cid=5,\n                    name=\"_city_id\",\n                    type=\"integer\",\n                    notnull=0,\n                    default_value=None,\n                    is_pk=0,\n                    hidden=0,\n                ),\n                Column(\n                    cid=6,\n                    name=\"_neighborhood\",\n                    type=\"text\",\n                    notnull=0,\n                    default_value=None,\n                    is_pk=0,\n                    hidden=0,\n                ),\n                Column(\n                    cid=7,\n                    name=\"tags\",\n                    type=\"text\",\n                    notnull=0,\n                    default_value=None,\n                    is_pk=0,\n                    hidden=0,\n                ),\n                Column(\n                    cid=8,\n                    name=\"complex_array\",\n                    type=\"text\",\n                    notnull=0,\n                    default_value=None,\n                    is_pk=0,\n                    hidden=0,\n                ),\n                Column(\n                    cid=9,\n                    name=\"distinct_some_null\",\n                    type=\"\",\n                    notnull=0,\n                    default_value=None,\n                    is_pk=0,\n                    hidden=0,\n                ),\n                Column(\n                    cid=10,\n                    name=\"n\",\n                    type=\"text\",\n                    notnull=0,\n                    default_value=None,\n                    is_pk=0,\n                    hidden=0,\n                ),\n            ],\n        ),\n        (\n            \"sortable\",\n            [\n                Column(\n                    cid=0,\n                    name=\"pk1\",\n                    type=\"varchar(30)\",\n                    notnull=0,\n                    default_value=None,\n                    is_pk=1,\n                    hidden=0,\n                ),\n                Column(\n                    cid=1,\n                    name=\"pk2\",\n                    type=\"varchar(30)\",\n                    notnull=0,\n                    default_value=None,\n                    is_pk=2,\n                    hidden=0,\n                ),\n                Column(\n                    cid=2,\n                    name=\"content\",\n                    type=\"text\",\n                    notnull=0,\n                    default_value=None,\n                    is_pk=0,\n                    hidden=0,\n                ),\n                Column(\n                    cid=3,\n                    name=\"sortable\",\n                    type=\"integer\",\n                    notnull=0,\n                    default_value=None,\n                    is_pk=0,\n                    hidden=0,\n                ),\n                Column(\n                    cid=4,\n                    name=\"sortable_with_nulls\",\n                    type=\"real\",\n                    notnull=0,\n                    default_value=None,\n                    is_pk=0,\n                    hidden=0,\n                ),\n                Column(\n                    cid=5,\n                    name=\"sortable_with_nulls_2\",\n                    type=\"real\",\n                    notnull=0,\n                    default_value=None,\n                    is_pk=0,\n                    hidden=0,\n                ),\n                Column(\n                    cid=6,\n                    name=\"text\",\n                    type=\"text\",\n                    notnull=0,\n                    default_value=None,\n                    is_pk=0,\n                    hidden=0,\n                ),\n            ],\n        ),\n    ),\n)\n@pytest.mark.asyncio\nasync def test_table_column_details(db, table, expected):\n    columns = await db.table_column_details(table)\n    # Convert \"type\" to lowercase before comparison\n    # https://github.com/simonw/datasette/issues/1647\n    compare_columns = [\n        Column(\n            c.cid, c.name, c.type.lower(), c.notnull, c.default_value, c.is_pk, c.hidden\n        )\n        for c in columns\n    ]\n    assert compare_columns == expected\n\n\n@pytest.mark.asyncio\nasync def test_get_all_foreign_keys(db):\n    all_foreign_keys = await db.get_all_foreign_keys()\n    assert all_foreign_keys[\"roadside_attraction_characteristics\"] == {\n        \"incoming\": [],\n        \"outgoing\": [\n            {\n                \"other_table\": \"attraction_characteristic\",\n                \"column\": \"characteristic_id\",\n                \"other_column\": \"pk\",\n            },\n            {\n                \"other_table\": \"roadside_attractions\",\n                \"column\": \"attraction_id\",\n                \"other_column\": \"pk\",\n            },\n        ],\n    }\n    assert all_foreign_keys[\"attraction_characteristic\"] == {\n        \"incoming\": [\n            {\n                \"other_table\": \"roadside_attraction_characteristics\",\n                \"column\": \"pk\",\n                \"other_column\": \"characteristic_id\",\n            }\n        ],\n        \"outgoing\": [],\n    }\n    assert all_foreign_keys[\"compound_primary_key\"] == {\n        # No incoming because these are compound foreign keys, which we currently ignore\n        \"incoming\": [],\n        \"outgoing\": [],\n    }\n    assert all_foreign_keys[\"foreign_key_references\"] == {\n        \"incoming\": [],\n        \"outgoing\": [\n            {\n                \"other_table\": \"primary_key_multiple_columns\",\n                \"column\": \"foreign_key_with_no_label\",\n                \"other_column\": \"id\",\n            },\n            {\n                \"other_table\": \"simple_primary_key\",\n                \"column\": \"foreign_key_with_blank_label\",\n                \"other_column\": \"id\",\n            },\n            {\n                \"other_table\": \"simple_primary_key\",\n                \"column\": \"foreign_key_with_label\",\n                \"other_column\": \"id\",\n            },\n        ],\n    }\n\n\n@pytest.mark.asyncio\nasync def test_table_names(db):\n    table_names = await db.table_names()\n    # Tables are sorted alphabetically by name\n    assert table_names == [\n        \"123_starts_with_digits\",\n        \"Table With Space In Name\",\n        \"attraction_characteristic\",\n        \"binary_data\",\n        \"complex_foreign_keys\",\n        \"compound_primary_key\",\n        \"compound_three_primary_keys\",\n        \"custom_foreign_key_label\",\n        \"facet_cities\",\n        \"facetable\",\n        \"foreign_key_references\",\n        \"infinity\",\n        \"no_primary_key\",\n        \"primary_key_multiple_columns\",\n        \"primary_key_multiple_columns_explicit_label\",\n        \"roadside_attraction_characteristics\",\n        \"roadside_attractions\",\n        \"searchable\",\n        \"searchable_fts\",\n        \"searchable_fts_config\",\n        \"searchable_fts_data\",\n        \"searchable_fts_docsize\",\n        \"searchable_fts_idx\",\n        \"searchable_tags\",\n        \"select\",\n        \"simple_primary_key\",\n        \"sortable\",\n        \"table/with/slashes.csv\",\n        \"tags\",\n    ]\n\n\n@pytest.mark.asyncio\nasync def test_view_names(db):\n    view_names = await db.view_names()\n    assert view_names == [\n        \"paginated_view\",\n        \"simple_view\",\n        \"searchable_view\",\n        \"searchable_view_configured_by_metadata\",\n    ]\n\n\n@pytest.mark.asyncio\nasync def test_execute_write_block_true(db):\n    await db.execute_write(\n        \"update roadside_attractions set name = ? where pk = ?\", [\"Mystery!\", 1]\n    )\n    rows = await db.execute(\"select name from roadside_attractions where pk = 1\")\n    assert \"Mystery!\" == rows.rows[0][0]\n\n\n@pytest.mark.asyncio\nasync def test_execute_write_block_false(db):\n    await db.execute_write(\n        \"update roadside_attractions set name = ? where pk = ?\",\n        [\"Mystery!\", 1],\n    )\n    time.sleep(0.1)\n    rows = await db.execute(\"select name from roadside_attractions where pk = 1\")\n    assert \"Mystery!\" == rows.rows[0][0]\n\n\n@pytest.mark.asyncio\nasync def test_execute_write_script(db):\n    await db.execute_write_script(\n        \"create table foo (id integer primary key); create table bar (id integer primary key);\"\n    )\n    table_names = await db.table_names()\n    assert {\"foo\", \"bar\"}.issubset(table_names)\n\n\n@pytest.mark.asyncio\nasync def test_execute_write_many(db):\n    await db.execute_write_script(\"create table foomany (id integer primary key)\")\n    await db.execute_write_many(\n        \"insert into foomany (id) values (?)\", [(1,), (10,), (100,)]\n    )\n    result = await db.execute(\"select * from foomany\")\n    assert [r[0] for r in result.rows] == [1, 10, 100]\n\n\n@pytest.mark.asyncio\nasync def test_execute_write_has_correctly_prepared_connection(db):\n    # The sleep() function is only available if ds._prepare_connection() was called\n    await db.execute_write(\"select sleep(0.01)\")\n\n\n@pytest.mark.asyncio\nasync def test_execute_write_fn_block_false(db):\n    def write_fn(conn):\n        conn.execute(\"delete from roadside_attractions where pk = 1;\")\n        row = conn.execute(\"select count(*) from roadside_attractions\").fetchone()\n        return row[0]\n\n    task_id = await db.execute_write_fn(write_fn, block=False)\n    assert isinstance(task_id, uuid.UUID)\n\n\n@pytest.mark.asyncio\nasync def test_execute_write_fn_block_true(db):\n    def write_fn(conn):\n        conn.execute(\"delete from roadside_attractions where pk = 1;\")\n        row = conn.execute(\"select count(*) from roadside_attractions\").fetchone()\n        return row[0]\n\n    new_count = await db.execute_write_fn(write_fn)\n    assert 3 == new_count\n\n\n@pytest.mark.asyncio\nasync def test_execute_write_fn_exception(db):\n    def write_fn(conn):\n        assert False\n\n    with pytest.raises(AssertionError):\n        await db.execute_write_fn(write_fn)\n\n\n@pytest.mark.asyncio\n@pytest.mark.timeout(1)\nasync def test_execute_write_fn_connection_exception(tmpdir, app_client):\n    path = str(tmpdir / \"immutable.db\")\n    sqlite3.connect(path).execute(\"vacuum\")\n    db = Database(app_client.ds, path=path, is_mutable=False)\n    app_client.ds.add_database(db, name=\"immutable-db\")\n\n    def write_fn(conn):\n        assert False\n\n    with pytest.raises(AssertionError):\n        await db.execute_write_fn(write_fn)\n\n    app_client.ds.remove_database(\"immutable-db\")\n\n\ndef table_exists(conn, name):\n    return bool(\n        conn.execute(\n            \"\"\"\n        with all_tables as (\n            select name from sqlite_master where type = 'table'\n                     union all\n            select name from temp.sqlite_master where type = 'table'\n        )\n        select 1 from all_tables where name = ?\n        \"\"\",\n            (name,),\n        ).fetchall(),\n    )\n\n\ndef table_exists_checker(name):\n    def inner(conn):\n        return table_exists(conn, name)\n\n    return inner\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\"disable_threads\", (False, True))\nasync def test_execute_isolated(db, disable_threads):\n    if disable_threads:\n        ds = Datasette(memory=True, settings={\"num_sql_threads\": 0})\n        db = ds.add_database(Database(ds, memory_name=\"test_num_sql_threads_zero\"))\n\n    # Create temporary table in write\n    await db.execute_write(\n        \"create temporary table created_by_write (id integer primary key)\"\n    )\n    # Should stay visible to write connection\n    assert await db.execute_write_fn(table_exists_checker(\"created_by_write\"))\n\n    def create_shared_table(conn):\n        conn.execute(\"create table shared (id integer primary key)\")\n        # And a temporary table that should not continue to exist\n        conn.execute(\n            \"create temporary table created_by_isolated (id integer primary key)\"\n        )\n        assert table_exists(conn, \"created_by_isolated\")\n        # Also confirm that created_by_write does not exist\n        return table_exists(conn, \"created_by_write\")\n\n    # shared should not exist\n    assert not await db.execute_fn(table_exists_checker(\"shared\"))\n\n    # Create it using isolated\n    created_by_write_exists = await db.execute_isolated_fn(create_shared_table)\n    assert not created_by_write_exists\n\n    # shared SHOULD exist now\n    assert await db.execute_fn(table_exists_checker(\"shared\"))\n\n    # created_by_isolated should not exist, even in write connection\n    assert not await db.execute_write_fn(table_exists_checker(\"created_by_isolated\"))\n\n    # ... and a second call to isolated should not see that connection either\n    assert not await db.execute_isolated_fn(table_exists_checker(\"created_by_isolated\"))\n\n\n@pytest.mark.asyncio\nasync def test_mtime_ns(db):\n    assert isinstance(db.mtime_ns, int)\n\n\ndef test_mtime_ns_is_none_for_memory(app_client):\n    memory_db = Database(app_client.ds, is_memory=True)\n    assert memory_db.is_memory is True\n    assert None is memory_db.mtime_ns\n\n\ndef test_is_mutable(app_client):\n    assert Database(app_client.ds, is_memory=True).is_mutable is True\n    assert Database(app_client.ds, is_memory=True, is_mutable=True).is_mutable is True\n    assert Database(app_client.ds, is_memory=True, is_mutable=False).is_mutable is False\n\n\n@pytest.mark.asyncio\nasync def test_attached_databases(app_client_two_attached_databases_crossdb_enabled):\n    database = app_client_two_attached_databases_crossdb_enabled.ds.get_database(\n        \"_memory\"\n    )\n    attached = await database.attached_databases()\n    assert {a.name for a in attached} == {\"extra database\", \"fixtures\"}\n\n\n@pytest.mark.asyncio\nasync def test_database_memory_name(app_client):\n    ds = app_client.ds\n    foo1 = ds.add_database(Database(ds, memory_name=\"foo\"))\n    foo2 = ds.add_memory_database(\"foo\")\n    bar1 = ds.add_database(Database(ds, memory_name=\"bar\"))\n    bar2 = ds.add_memory_database(\"bar\")\n    for db in (foo1, foo2, bar1, bar2):\n        table_names = await db.table_names()\n        assert table_names == []\n    # Now create a table in foo\n    await foo1.execute_write(\"create table foo (t text)\")\n    assert await foo1.table_names() == [\"foo\"]\n    assert await foo2.table_names() == [\"foo\"]\n    assert await bar1.table_names() == []\n    assert await bar2.table_names() == []\n\n\n@pytest.mark.asyncio\nasync def test_in_memory_databases_forbid_writes(app_client):\n    ds = app_client.ds\n    db = ds.add_database(Database(ds, memory_name=\"test\"))\n    with pytest.raises(sqlite3.OperationalError):\n        await db.execute(\"create table foo (t text)\")\n    assert await db.table_names() == []\n    # Using db.execute_write() should work:\n    await db.execute_write(\"create table foo (t text)\")\n    assert await db.table_names() == [\"foo\"]\n\n\ndef pragma_table_list_supported():\n    return sqlite_version()[1] >= 37\n\n\n@pytest.mark.asyncio\n@pytest.mark.skipif(\n    not pragma_table_list_supported(), reason=\"Requires PRAGMA table_list support\"\n)\nasync def test_hidden_tables(app_client):\n    ds = app_client.ds\n    db = ds.add_database(Database(ds, is_memory=True, is_mutable=True))\n    assert await db.hidden_table_names() == []\n    await db.execute(\"create virtual table f using fts5(a)\")\n    assert await db.hidden_table_names() == [\n        \"f_config\",\n        \"f_content\",\n        \"f_data\",\n        \"f_docsize\",\n        \"f_idx\",\n    ]\n\n    await db.execute(\"create virtual table r using rtree(id, amin, amax)\")\n    assert await db.hidden_table_names() == [\n        \"f_config\",\n        \"f_content\",\n        \"f_data\",\n        \"f_docsize\",\n        \"f_idx\",\n        \"r_node\",\n        \"r_parent\",\n        \"r_rowid\",\n    ]\n\n    await db.execute(\"create table _hideme(_)\")\n    assert await db.hidden_table_names() == [\n        \"_hideme\",\n        \"f_config\",\n        \"f_content\",\n        \"f_data\",\n        \"f_docsize\",\n        \"f_idx\",\n        \"r_node\",\n        \"r_parent\",\n        \"r_rowid\",\n    ]\n\n    # A fts virtual table with a content table should be hidden too\n    await db.execute(\"create virtual table f2_fts using fts5(a, content='f')\")\n    assert await db.hidden_table_names() == [\n        \"_hideme\",\n        \"f2_fts_config\",\n        \"f2_fts_data\",\n        \"f2_fts_docsize\",\n        \"f2_fts_idx\",\n        \"f_config\",\n        \"f_content\",\n        \"f_data\",\n        \"f_docsize\",\n        \"f_idx\",\n        \"r_node\",\n        \"r_parent\",\n        \"r_rowid\",\n        \"f2_fts\",\n    ]\n\n\n@pytest.mark.asyncio\nasync def test_replace_database(tmpdir):\n    path1 = str(tmpdir / \"data1.db\")\n    (tmpdir / \"two\").mkdir()\n    path2 = str(tmpdir / \"two\" / \"data1.db\")\n    sqlite3.connect(path1).executescript(\"\"\"\n        create table t (id integer primary key);\n        insert into t (id) values (1);\n        insert into t (id) values (2);\n    \"\"\")\n    sqlite3.connect(path2).executescript(\"\"\"\n        create table t (id integer primary key);\n        insert into t (id) values (1);\n    \"\"\")\n    datasette = Datasette([path1])\n    db = datasette.get_database(\"data1\")\n    count = (await db.execute(\"select count(*) from t\")).first()[0]\n    assert count == 2\n    # Now replace that database\n    datasette.get_database(\"data1\").close()\n    datasette.remove_database(\"data1\")\n    datasette.add_database(Database(datasette, path2), \"data1\")\n    db2 = datasette.get_database(\"data1\")\n    count = (await db2.execute(\"select count(*) from t\")).first()[0]\n    assert count == 1\n"
  },
  {
    "path": "tests/test_internals_datasette.py",
    "content": "\"\"\"\nTests for the datasette.app.Datasette class\n\"\"\"\n\nimport dataclasses\nfrom datasette import Context\nfrom datasette.app import Datasette, Database, ResourcesSQL\nfrom datasette.resources import DatabaseResource\nfrom itsdangerous import BadSignature\nimport pytest\n\n\n@pytest.fixture\ndef datasette(ds_client):\n    return ds_client.ds\n\n\ndef test_get_database(datasette):\n    db = datasette.get_database(\"fixtures\")\n    assert \"fixtures\" == db.name\n    with pytest.raises(KeyError):\n        datasette.get_database(\"missing\")\n\n\ndef test_get_database_no_argument(datasette):\n    # Returns the first available database:\n    db = datasette.get_database()\n    assert \"fixtures\" == db.name\n\n\n@pytest.mark.parametrize(\"value\", [\"hello\", 123, {\"key\": \"value\"}])\n@pytest.mark.parametrize(\"namespace\", [None, \"two\"])\ndef test_sign_unsign(datasette, value, namespace):\n    extra_args = [namespace] if namespace else []\n    signed = datasette.sign(value, *extra_args)\n    assert value != signed\n    assert value == datasette.unsign(signed, *extra_args)\n    with pytest.raises(BadSignature):\n        datasette.unsign(signed[:-1] + (\"!\" if signed[-1] != \"!\" else \":\"))\n\n\n@pytest.mark.parametrize(\n    \"setting,expected\",\n    (\n        (\"base_url\", \"/\"),\n        (\"max_csv_mb\", 100),\n        (\"allow_csv_stream\", True),\n    ),\n)\ndef test_datasette_setting(datasette, setting, expected):\n    assert datasette.setting(setting) == expected\n\n\n@pytest.mark.asyncio\nasync def test_datasette_constructor():\n    ds = Datasette()\n    databases = (await ds.client.get(\"/-/databases.json\")).json()\n    assert databases == [\n        {\n            \"name\": \"_memory\",\n            \"route\": \"_memory\",\n            \"path\": None,\n            \"size\": 0,\n            \"is_mutable\": False,\n            \"is_memory\": True,\n            \"hash\": None,\n        }\n    ]\n\n\n@pytest.mark.asyncio\nasync def test_num_sql_threads_zero():\n    ds = Datasette([], memory=True, settings={\"num_sql_threads\": 0})\n    db = ds.add_database(Database(ds, memory_name=\"test_num_sql_threads_zero\"))\n    await db.execute_write(\"create table t(id integer primary key)\")\n    await db.execute_write(\"insert into t (id) values (1)\")\n    response = await ds.client.get(\"/-/threads.json\")\n    assert response.json() == {\"num_threads\": 0, \"threads\": []}\n    response2 = await ds.client.get(\"/test_num_sql_threads_zero/t.json?_shape=array\")\n    assert response2.json() == [{\"id\": 1}]\n\n\nROOT = {\"id\": \"root\"}\nALLOW_ROOT = {\"allow\": {\"id\": \"root\"}}\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"actor,config,action,resource,should_allow,expected_private\",\n    (\n        (None, ALLOW_ROOT, \"view-instance\", None, False, False),\n        (ROOT, ALLOW_ROOT, \"view-instance\", None, True, True),\n        (\n            None,\n            {\"databases\": {\"_memory\": ALLOW_ROOT}},\n            \"view-database\",\n            DatabaseResource(database=\"_memory\"),\n            False,\n            False,\n        ),\n        (\n            ROOT,\n            {\"databases\": {\"_memory\": ALLOW_ROOT}},\n            \"view-database\",\n            DatabaseResource(database=\"_memory\"),\n            True,\n            True,\n        ),\n        # Check private is false for non-protected instance check\n        (\n            ROOT,\n            {\"allow\": True},\n            \"view-instance\",\n            None,\n            True,\n            False,\n        ),\n    ),\n)\nasync def test_datasette_check_visibility(\n    actor, config, action, resource, should_allow, expected_private\n):\n    ds = Datasette([], memory=True, config=config)\n    await ds.invoke_startup()\n    visible, private = await ds.check_visibility(\n        actor, action=action, resource=resource\n    )\n    assert visible == should_allow\n    assert private == expected_private\n\n\n@pytest.mark.asyncio\nasync def test_datasette_render_template_no_request():\n    # https://github.com/simonw/datasette/issues/1849\n    ds = Datasette(memory=True)\n    await ds.invoke_startup()\n    rendered = await ds.render_template(\"error.html\")\n    assert \"Error \" in rendered\n\n\n@pytest.mark.asyncio\nasync def test_datasette_render_template_with_dataclass():\n    @dataclasses.dataclass\n    class ExampleContext(Context):\n        title: str\n        status: int\n        error: str\n\n    context = ExampleContext(title=\"Hello\", status=200, error=\"Error message\")\n    ds = Datasette(memory=True)\n    await ds.invoke_startup()\n    rendered = await ds.render_template(\"error.html\", context)\n    assert \"
    Hello
    \" in rendered\n    assert \"Error message\" in rendered\n\n\ndef test_datasette_error_if_string_not_list(tmpdir):\n    # https://github.com/simonw/datasette/issues/1985\n    db_path = str(tmpdir / \"data.db\")\n    with pytest.raises(ValueError):\n        Datasette(db_path)\n\n\n@pytest.mark.asyncio\nasync def test_get_action(ds_client):\n    ds = ds_client.ds\n    for name_or_abbr in (\n        \"vi\",\n        \"view-instance\",\n        \"vt\",\n        \"view-table\",\n        \"sct\",\n        \"set-column-type\",\n    ):\n        action = ds.get_action(name_or_abbr)\n        if \"-\" in name_or_abbr:\n            assert action.name == name_or_abbr\n        else:\n            assert action.abbr == name_or_abbr\n    # And test None return for missing action\n    assert ds.get_action(\"missing-permission\") is None\n\n\n@pytest.mark.asyncio\nasync def test_apply_metadata_json():\n    ds = Datasette(\n        metadata={\n            \"databases\": {\n                \"legislators\": {\n                    \"tables\": {\"offices\": {\"summary\": \"office address or sumtin\"}},\n                    \"queries\": {\n                        \"millennial_representatives\": {\n                            \"summary\": \"Social media accounts for current legislators\"\n                        }\n                    },\n                }\n            },\n            \"weird_instance_value\": {\"nested\": [1, 2, 3]},\n        },\n    )\n    await ds.invoke_startup()\n    assert (await ds.client.get(\"/\")).status_code == 200\n    value = (await ds.get_instance_metadata()).get(\"weird_instance_value\")\n    assert value == '{\"nested\": [1, 2, 3]}'\n\n\n@pytest.mark.asyncio\nasync def test_allowed_resources_sql(datasette):\n    result = await datasette.allowed_resources_sql(\n        action=\"view-table\",\n        actor=None,\n    )\n    assert isinstance(result, ResourcesSQL)\n    assert \"all_rules AS\" in result.sql\n    assert result.params[\"action\"] == \"view-table\"\n"
  },
  {
    "path": "tests/test_internals_datasette_client.py",
    "content": "import httpx\nimport pytest\nimport pytest_asyncio\nfrom datasette.app import Datasette\n\n\n@pytest_asyncio.fixture\nasync def datasette(ds_client):\n    await ds_client.ds.invoke_startup()\n    return ds_client.ds\n\n\n@pytest_asyncio.fixture\nasync def datasette_with_permissions():\n    \"\"\"A datasette instance with permission restrictions for testing\"\"\"\n    ds = Datasette(config={\"databases\": {\"test_db\": {\"allow\": {\"id\": \"admin\"}}}})\n    await ds.invoke_startup()\n    db = ds.add_memory_database(\"test_datasette_with_permissions\", name=\"test_db\")\n    await db.execute_write(\n        \"create table if not exists test_table (id integer primary key, name text)\"\n    )\n    await db.execute_write(\n        \"insert or ignore into test_table (id, name) values (1, 'Alice')\"\n    )\n    # Trigger catalog refresh\n    await ds.client.get(\"/\")\n    return ds\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"method,path,expected_status\",\n    [\n        (\"get\", \"/\", 200),\n        (\"options\", \"/\", 200),\n        (\"head\", \"/\", 200),\n        (\"put\", \"/\", 405),\n        (\"patch\", \"/\", 405),\n        (\"delete\", \"/\", 405),\n    ],\n)\nasync def test_client_methods(datasette, method, path, expected_status):\n    client_method = getattr(datasette.client, method)\n    response = await client_method(path)\n    assert isinstance(response, httpx.Response)\n    assert response.status_code == expected_status\n    # Try that again using datasette.client.request\n    response2 = await datasette.client.request(method, path)\n    assert response2.status_code == expected_status\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\"prefix\", [None, \"/prefix/\"])\nasync def test_client_post(datasette, prefix):\n    original_base_url = datasette._settings[\"base_url\"]\n    try:\n        if prefix is not None:\n            datasette._settings[\"base_url\"] = prefix\n        response = await datasette.client.post(\n            \"/-/messages\",\n            data={\n                \"message\": \"A message\",\n            },\n        )\n        assert isinstance(response, httpx.Response)\n        assert response.status_code == 302\n        assert \"ds_messages\" in response.cookies\n    finally:\n        datasette._settings[\"base_url\"] = original_base_url\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"prefix,expected_path\", [(None, \"/asgi-scope\"), (\"/prefix/\", \"/prefix/asgi-scope\")]\n)\nasync def test_client_path(datasette, prefix, expected_path):\n    original_base_url = datasette._settings[\"base_url\"]\n    try:\n        if prefix is not None:\n            datasette._settings[\"base_url\"] = prefix\n        response = await datasette.client.get(\"/asgi-scope\")\n        path = response.json()[\"path\"]\n        assert path == expected_path\n    finally:\n        datasette._settings[\"base_url\"] = original_base_url\n\n\n@pytest.mark.asyncio\nasync def test_skip_permission_checks_allows_forbidden_access(\n    datasette_with_permissions,\n):\n    \"\"\"Test that skip_permission_checks=True bypasses permission checks\"\"\"\n    ds = datasette_with_permissions\n\n    # Without skip_permission_checks, anonymous user should get 403 for protected database\n    response = await ds.client.get(\"/test_db.json\")\n    assert response.status_code == 403\n\n    # With skip_permission_checks=True, should get 200\n    response = await ds.client.get(\"/test_db.json\", skip_permission_checks=True)\n    assert response.status_code == 200\n    data = response.json()\n    assert data[\"database\"] == \"test_db\"\n\n\n@pytest.mark.asyncio\nasync def test_skip_permission_checks_on_table(datasette_with_permissions):\n    \"\"\"Test skip_permission_checks works for table access\"\"\"\n    ds = datasette_with_permissions\n\n    # Without skip_permission_checks, should get 403\n    response = await ds.client.get(\"/test_db/test_table.json\")\n    assert response.status_code == 403\n\n    # With skip_permission_checks=True, should get table data\n    response = await ds.client.get(\n        \"/test_db/test_table.json\", skip_permission_checks=True\n    )\n    assert response.status_code == 200\n    data = response.json()\n    assert data[\"rows\"] == [{\"id\": 1, \"name\": \"Alice\"}]\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"method\", [\"get\", \"post\", \"put\", \"patch\", \"delete\", \"options\", \"head\"]\n)\nasync def test_skip_permission_checks_all_methods(datasette_with_permissions, method):\n    \"\"\"Test that skip_permission_checks works with all HTTP methods\"\"\"\n    ds = datasette_with_permissions\n\n    # All methods should work with skip_permission_checks=True\n    client_method = getattr(ds.client, method)\n    response = await client_method(\"/test_db.json\", skip_permission_checks=True)\n    # We don't check status code since some methods might not be allowed,\n    # but we verify the request doesn't fail due to permissions\n    assert isinstance(response, httpx.Response)\n\n\n@pytest.mark.asyncio\nasync def test_skip_permission_checks_request_method(datasette_with_permissions):\n    \"\"\"Test that skip_permission_checks works with client.request()\"\"\"\n    ds = datasette_with_permissions\n\n    # Without skip_permission_checks\n    response = await ds.client.request(\"GET\", \"/test_db.json\")\n    assert response.status_code == 403\n\n    # With skip_permission_checks=True\n    response = await ds.client.request(\n        \"GET\", \"/test_db.json\", skip_permission_checks=True\n    )\n    assert response.status_code == 200\n\n\n@pytest.mark.asyncio\nasync def test_skip_permission_checks_isolated_to_request(datasette_with_permissions):\n    \"\"\"Test that skip_permission_checks doesn't affect other concurrent requests\"\"\"\n    ds = datasette_with_permissions\n\n    # First request with skip_permission_checks=True should succeed\n    response1 = await ds.client.get(\"/test_db.json\", skip_permission_checks=True)\n    assert response1.status_code == 200\n\n    # Subsequent request without it should still get 403\n    response2 = await ds.client.get(\"/test_db.json\")\n    assert response2.status_code == 403\n\n    # And another with skip should succeed again\n    response3 = await ds.client.get(\"/test_db.json\", skip_permission_checks=True)\n    assert response3.status_code == 200\n\n\n@pytest.mark.asyncio\nasync def test_skip_permission_checks_with_admin_actor(datasette_with_permissions):\n    \"\"\"Test that skip_permission_checks works even when actor is provided\"\"\"\n    ds = datasette_with_permissions\n\n    # Admin actor should normally have access\n    admin_cookies = {\"ds_actor\": ds.client.actor_cookie({\"id\": \"admin\"})}\n    response = await ds.client.get(\"/test_db.json\", cookies=admin_cookies)\n    assert response.status_code == 200\n\n    # Non-admin actor should get 403\n    user_cookies = {\"ds_actor\": ds.client.actor_cookie({\"id\": \"user\"})}\n    response = await ds.client.get(\"/test_db.json\", cookies=user_cookies)\n    assert response.status_code == 403\n\n    # Non-admin actor with skip_permission_checks=True should get 200\n    response = await ds.client.get(\n        \"/test_db.json\", cookies=user_cookies, skip_permission_checks=True\n    )\n    assert response.status_code == 200\n\n\n@pytest.mark.asyncio\nasync def test_skip_permission_checks_shows_denied_tables():\n    \"\"\"Test that skip_permission_checks=True shows tables from denied databases in /-/tables.json\"\"\"\n    ds = Datasette(\n        config={\n            \"databases\": {\n                \"fixtures\": {\"allow\": False}  # Deny all access to this database\n            }\n        }\n    )\n    await ds.invoke_startup()\n    db = ds.add_memory_database(\"fixtures\")\n    await db.execute_write(\n        \"CREATE TABLE test_table (id INTEGER PRIMARY KEY, name TEXT)\"\n    )\n    await db.execute_write(\"INSERT INTO test_table (id, name) VALUES (1, 'Alice')\")\n    await ds._refresh_schemas()\n\n    # Without skip_permission_checks, tables from denied database should not appear in /-/tables.json\n    response = await ds.client.get(\"/-/tables.json\")\n    assert response.status_code == 200\n    data = response.json()\n    table_names = [match[\"name\"] for match in data[\"matches\"]]\n    # Should not see any fixtures tables since access is denied\n    fixtures_tables = [name for name in table_names if name.startswith(\"fixtures:\")]\n    assert len(fixtures_tables) == 0\n\n    # With skip_permission_checks=True, tables from denied database SHOULD appear\n    response = await ds.client.get(\"/-/tables.json\", skip_permission_checks=True)\n    assert response.status_code == 200\n    data = response.json()\n    table_names = [match[\"name\"] for match in data[\"matches\"]]\n    # Should see fixtures tables when permission checks are skipped\n    assert \"fixtures: test_table\" in table_names\n\n\n@pytest.mark.asyncio\nasync def test_in_client_returns_false_outside_request(datasette):\n    \"\"\"Test that datasette.in_client() returns False outside of a client request\"\"\"\n    assert datasette.in_client() is False\n\n\n@pytest.mark.asyncio\nasync def test_in_client_returns_true_inside_request():\n    \"\"\"Test that datasette.in_client() returns True inside a client request\"\"\"\n    from datasette import hookimpl, Response\n\n    class TestPlugin:\n        __name__ = \"test_in_client_plugin\"\n\n        @hookimpl\n        def register_routes(self):\n            async def test_view(datasette):\n                # Assert in_client() returns True within the view\n                assert datasette.in_client() is True\n                return Response.json({\"in_client\": datasette.in_client()})\n\n            return [\n                (r\"^/-/test-in-client$\", test_view),\n            ]\n\n    ds = Datasette()\n    await ds.invoke_startup()\n    ds.pm.register(TestPlugin(), name=\"test_in_client_plugin\")\n    try:\n\n        # Outside of a client request, should be False\n        assert ds.in_client() is False\n\n        # Make a request via datasette.client\n        response = await ds.client.get(\"/-/test-in-client\")\n        assert response.status_code == 200\n        assert response.json()[\"in_client\"] is True\n\n        # After the request, should be False again\n        assert ds.in_client() is False\n    finally:\n        ds.pm.unregister(name=\"test_in_client_plugin\")\n\n\n@pytest.mark.asyncio\nasync def test_in_client_with_skip_permission_checks():\n    \"\"\"Test that in_client() works regardless of skip_permission_checks value\"\"\"\n    from datasette import hookimpl\n    from datasette.utils.asgi import Response\n\n    in_client_values = []\n\n    class TestPlugin:\n        __name__ = \"test_in_client_skip_plugin\"\n\n        @hookimpl\n        def register_routes(self):\n            async def test_view(datasette):\n                in_client_values.append(datasette.in_client())\n                return Response.json({\"in_client\": datasette.in_client()})\n\n            return [\n                (r\"^/-/test-in-client$\", test_view),\n            ]\n\n    ds = Datasette(config={\"databases\": {\"test_db\": {\"allow\": {\"id\": \"admin\"}}}})\n    await ds.invoke_startup()\n    ds.pm.register(TestPlugin(), name=\"test_in_client_skip_plugin\")\n    try:\n\n        # Request without skip_permission_checks\n        await ds.client.get(\"/-/test-in-client\")\n        # Request with skip_permission_checks=True\n        await ds.client.get(\"/-/test-in-client\", skip_permission_checks=True)\n\n        # Both should have detected in_client as True\n        assert (\n            len(in_client_values) == 2\n        ), f\"Expected 2 values, got {len(in_client_values)}\"\n        assert all(in_client_values), f\"Expected all True, got {in_client_values}\"\n    finally:\n        ds.pm.unregister(name=\"test_in_client_skip_plugin\")\n"
  },
  {
    "path": "tests/test_internals_request.py",
    "content": "from datasette.utils.asgi import Request\nimport json\nimport pytest\n\n\n@pytest.mark.asyncio\nasync def test_request_post_vars():\n    scope = {\n        \"http_version\": \"1.1\",\n        \"method\": \"POST\",\n        \"path\": \"/\",\n        \"raw_path\": b\"/\",\n        \"query_string\": b\"\",\n        \"scheme\": \"http\",\n        \"type\": \"http\",\n        \"headers\": [[b\"content-type\", b\"application/x-www-form-urlencoded\"]],\n    }\n\n    async def receive():\n        return {\n            \"type\": \"http.request\",\n            \"body\": b\"foo=bar&baz=1&empty=\",\n            \"more_body\": False,\n        }\n\n    request = Request(scope, receive)\n    assert {\"foo\": \"bar\", \"baz\": \"1\", \"empty\": \"\"} == await request.post_vars()\n\n\n@pytest.mark.asyncio\nasync def test_request_post_body():\n    scope = {\n        \"http_version\": \"1.1\",\n        \"method\": \"POST\",\n        \"path\": \"/\",\n        \"raw_path\": b\"/\",\n        \"query_string\": b\"\",\n        \"scheme\": \"http\",\n        \"type\": \"http\",\n        \"headers\": [[b\"content-type\", b\"application/json\"]],\n    }\n\n    data = {\"hello\": \"world\"}\n\n    async def receive():\n        return {\n            \"type\": \"http.request\",\n            \"body\": json.dumps(data, indent=4).encode(\"utf-8\"),\n            \"more_body\": False,\n        }\n\n    request = Request(scope, receive)\n    body = await request.post_body()\n    assert isinstance(body, bytes)\n    assert data == json.loads(body)\n\n\ndef test_request_args():\n    request = Request.fake(\"/foo?multi=1&multi=2&single=3\")\n    assert \"1\" == request.args.get(\"multi\")\n    assert \"3\" == request.args.get(\"single\")\n    assert \"1\" == request.args[\"multi\"]\n    assert \"3\" == request.args[\"single\"]\n    assert [\"1\", \"2\"] == request.args.getlist(\"multi\")\n    assert [] == request.args.getlist(\"missing\")\n    assert \"multi\" in request.args\n    assert \"single\" in request.args\n    assert \"missing\" not in request.args\n    expected = [\"multi\", \"single\"]\n    assert expected == list(request.args.keys())\n    for i, key in enumerate(request.args):\n        assert expected[i] == key\n    assert 2 == len(request.args)\n    with pytest.raises(KeyError):\n        request.args[\"missing\"]\n\n\ndef test_request_fake_url_vars():\n    request = Request.fake(\"/\")\n    assert request.url_vars == {}\n    request = Request.fake(\"/\", url_vars={\"database\": \"fixtures\"})\n    assert request.url_vars == {\"database\": \"fixtures\"}\n\n\ndef test_request_repr():\n    request = Request.fake(\"/foo?multi=1&multi=2&single=3\")\n    assert (\n        repr(request)\n        == ''\n    )\n\n\ndef test_request_url_vars():\n    scope = {\n        \"http_version\": \"1.1\",\n        \"method\": \"POST\",\n        \"path\": \"/\",\n        \"raw_path\": b\"/\",\n        \"query_string\": b\"\",\n        \"scheme\": \"http\",\n        \"type\": \"http\",\n        \"headers\": [[b\"content-type\", b\"application/x-www-form-urlencoded\"]],\n    }\n    assert {} == Request(scope, None).url_vars\n    assert {\"name\": \"cleo\"} == Request(\n        dict(scope, url_route={\"kwargs\": {\"name\": \"cleo\"}}), None\n    ).url_vars\n\n\n@pytest.mark.parametrize(\n    \"path,query_string,expected_full_path\",\n    [(\"/\", \"\", \"/\"), (\"/\", \"foo=bar\", \"/?foo=bar\"), (\"/foo\", \"bar\", \"/foo?bar\")],\n)\ndef test_request_properties(path, query_string, expected_full_path):\n    path_with_query_string = path\n    if query_string:\n        path_with_query_string += \"?\" + query_string\n    scope = {\n        \"http_version\": \"1.1\",\n        \"method\": \"POST\",\n        \"path\": path,\n        \"raw_path\": path_with_query_string.encode(\"latin-1\"),\n        \"query_string\": query_string.encode(\"latin-1\"),\n        \"scheme\": \"http\",\n        \"type\": \"http\",\n    }\n    request = Request(scope, None)\n    assert request.path == path\n    assert request.query_string == query_string\n    assert request.full_path == expected_full_path\n\n\ndef test_request_blank_values():\n    request = Request.fake(\"/?a=b&foo=bar&foo=bar2&baz=\")\n    assert request.args._data == {\"a\": [\"b\"], \"foo\": [\"bar\", \"bar2\"], \"baz\": [\"\"]}\n\n\ndef test_json_in_query_string_name():\n    query_string = (\n        '?_through.[\"roadside_attraction_characteristics\"%2C\"characteristic_id\"]=1'\n    )\n    request = Request.fake(\"/\" + query_string)\n    assert (\n        request.args[\n            '_through.[\"roadside_attraction_characteristics\",\"characteristic_id\"]'\n        ]\n        == \"1\"\n    )\n"
  },
  {
    "path": "tests/test_internals_response.py",
    "content": "from datasette.utils.asgi import Response\nimport pytest\n\n\ndef test_response_html():\n    response = Response.html(\"Hello from HTML\")\n    assert 200 == response.status\n    assert \"Hello from HTML\" == response.body\n    assert \"text/html; charset=utf-8\" == response.content_type\n\n\ndef test_response_text():\n    response = Response.text(\"Hello from text\")\n    assert 200 == response.status\n    assert \"Hello from text\" == response.body\n    assert \"text/plain; charset=utf-8\" == response.content_type\n\n\ndef test_response_json():\n    response = Response.json({\"this_is\": \"json\"})\n    assert 200 == response.status\n    assert '{\"this_is\": \"json\"}' == response.body\n    assert \"application/json; charset=utf-8\" == response.content_type\n\n\ndef test_response_redirect():\n    response = Response.redirect(\"/foo\")\n    assert 302 == response.status\n    assert \"/foo\" == response.headers[\"Location\"]\n\n\n@pytest.mark.asyncio\nasync def test_response_set_cookie():\n    events = []\n\n    async def send(event):\n        events.append(event)\n\n    response = Response.redirect(\"/foo\")\n    response.set_cookie(\"foo\", \"bar\", max_age=10, httponly=True)\n    await response.asgi_send(send)\n\n    assert [\n        {\n            \"type\": \"http.response.start\",\n            \"status\": 302,\n            \"headers\": [\n                [b\"Location\", b\"/foo\"],\n                [b\"content-type\", b\"text/plain\"],\n                [b\"set-cookie\", b\"foo=bar; HttpOnly; Max-Age=10; Path=/; SameSite=lax\"],\n            ],\n        },\n        {\"type\": \"http.response.body\", \"body\": b\"\"},\n    ] == events\n"
  },
  {
    "path": "tests/test_internals_urls.py",
    "content": "from datasette.app import Datasette\nfrom datasette.utils import PrefixedUrlString\nimport pytest\n\n\n@pytest.fixture(scope=\"module\")\ndef ds():\n    return Datasette([], memory=True)\n\n\n@pytest.mark.parametrize(\n    \"base_url,path,expected\",\n    [\n        (\"/\", \"/\", \"/\"),\n        (\"/\", \"/foo\", \"/foo\"),\n        (\"/prefix/\", \"/\", \"/prefix/\"),\n        (\"/prefix/\", \"/foo\", \"/prefix/foo\"),\n        (\"/prefix/\", \"foo\", \"/prefix/foo\"),\n    ],\n)\ndef test_path(ds, base_url, path, expected):\n    ds._settings[\"base_url\"] = base_url\n    actual = ds.urls.path(path)\n    assert actual == expected\n    assert isinstance(actual, PrefixedUrlString)\n\n\ndef test_path_applied_twice_does_not_double_prefix(ds):\n    ds._settings[\"base_url\"] = \"/prefix/\"\n    path = ds.urls.path(\"/\")\n    assert path == \"/prefix/\"\n    path = ds.urls.path(path)\n    assert path == \"/prefix/\"\n\n\n@pytest.mark.parametrize(\n    \"base_url,expected\",\n    [\n        (\"/\", \"/\"),\n        (\"/prefix/\", \"/prefix/\"),\n    ],\n)\ndef test_instance(ds, base_url, expected):\n    ds._settings[\"base_url\"] = base_url\n    actual = ds.urls.instance()\n    assert actual == expected\n    assert isinstance(actual, PrefixedUrlString)\n\n\n@pytest.mark.parametrize(\n    \"base_url,file,expected\",\n    [\n        (\"/\", \"foo.js\", \"/-/static/foo.js\"),\n        (\"/prefix/\", \"foo.js\", \"/prefix/-/static/foo.js\"),\n    ],\n)\ndef test_static(ds, base_url, file, expected):\n    ds._settings[\"base_url\"] = base_url\n    actual = ds.urls.static(file)\n    assert actual == expected\n    assert isinstance(actual, PrefixedUrlString)\n\n\n@pytest.mark.parametrize(\n    \"base_url,plugin,file,expected\",\n    [\n        (\n            \"/\",\n            \"datasette_cluster_map\",\n            \"datasette-cluster-map.js\",\n            \"/-/static-plugins/datasette_cluster_map/datasette-cluster-map.js\",\n        ),\n        (\n            \"/prefix/\",\n            \"datasette_cluster_map\",\n            \"datasette-cluster-map.js\",\n            \"/prefix/-/static-plugins/datasette_cluster_map/datasette-cluster-map.js\",\n        ),\n    ],\n)\ndef test_static_plugins(ds, base_url, plugin, file, expected):\n    ds._settings[\"base_url\"] = base_url\n    actual = ds.urls.static_plugins(plugin, file)\n    assert actual == expected\n    assert isinstance(actual, PrefixedUrlString)\n\n\n@pytest.mark.parametrize(\n    \"base_url,expected\",\n    [\n        (\"/\", \"/-/logout\"),\n        (\"/prefix/\", \"/prefix/-/logout\"),\n    ],\n)\ndef test_logout(ds, base_url, expected):\n    ds._settings[\"base_url\"] = base_url\n    actual = ds.urls.logout()\n    assert actual == expected\n    assert isinstance(actual, PrefixedUrlString)\n\n\n@pytest.mark.parametrize(\n    \"base_url,format,expected\",\n    [\n        (\"/\", None, \"/_memory\"),\n        (\"/prefix/\", None, \"/prefix/_memory\"),\n        (\"/\", \"json\", \"/_memory.json\"),\n    ],\n)\ndef test_database(ds, base_url, format, expected):\n    ds._settings[\"base_url\"] = base_url\n    actual = ds.urls.database(\"_memory\", format=format)\n    assert actual == expected\n    assert isinstance(actual, PrefixedUrlString)\n\n\n@pytest.mark.parametrize(\n    \"base_url,name,format,expected\",\n    [\n        (\"/\", \"name\", None, \"/_memory/name\"),\n        (\"/prefix/\", \"name\", None, \"/prefix/_memory/name\"),\n        (\"/\", \"name\", \"json\", \"/_memory/name.json\"),\n        (\"/\", \"name.json\", \"json\", \"/_memory/name~2Ejson.json\"),\n    ],\n)\ndef test_table_and_query(ds, base_url, name, format, expected):\n    ds._settings[\"base_url\"] = base_url\n    actual1 = ds.urls.table(\"_memory\", name, format=format)\n    assert actual1 == expected\n    assert isinstance(actual1, PrefixedUrlString)\n    actual2 = ds.urls.query(\"_memory\", name, format=format)\n    assert actual2 == expected\n    assert isinstance(actual2, PrefixedUrlString)\n\n\n@pytest.mark.parametrize(\n    \"base_url,format,expected\",\n    [\n        (\"/\", None, \"/_memory/facetable/1\"),\n        (\"/prefix/\", None, \"/prefix/_memory/facetable/1\"),\n        (\"/\", \"json\", \"/_memory/facetable/1.json\"),\n    ],\n)\ndef test_row(ds, base_url, format, expected):\n    ds._settings[\"base_url\"] = base_url\n    actual = ds.urls.row(\"_memory\", \"facetable\", \"1\", format=format)\n    assert actual == expected\n    assert isinstance(actual, PrefixedUrlString)\n"
  },
  {
    "path": "tests/test_label_column_for_table.py",
    "content": "import pytest\nfrom datasette.database import Database\nfrom datasette.app import Datasette\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"create_sql,table_name,config,expected_label_column\",\n    [\n        # Explicit label_column\n        (\n            \"create table t1 (id integer primary key, name text, title text);\",\n            \"t1\",\n            {\"t1\": {\"label_column\": \"title\"}},\n            \"title\",\n        ),\n        # Single unique text column\n        (\n            \"create table t2 (id integer primary key, name2 text unique, title text);\",\n            \"t2\",\n            {},\n            \"name2\",\n        ),\n        (\n            \"create table t3 (id integer primary key, title2 text unique, name text);\",\n            \"t3\",\n            {},\n            \"title2\",\n        ),\n        # Two unique text columns means it cannot decide on one\n        (\n            \"create table t3x (id integer primary key, name2 text unique, title2 text unique);\",\n            \"t3x\",\n            {},\n            None,\n        ),\n        # Name or title column\n        (\n            \"create table t4 (id integer primary key, name text);\",\n            \"t4\",\n            {},\n            \"name\",\n        ),\n        (\n            \"create table t5 (id integer primary key, title text);\",\n            \"t5\",\n            {},\n            \"title\",\n        ),\n        # But not if there are multiple non-unique text that are not called title\n        (\n            \"create table t5x (id integer primary key, other1 text, other2 text);\",\n            \"t5x\",\n            {},\n            None,\n        ),\n        (\n            \"create table t6 (id integer primary key, Name text);\",\n            \"t6\",\n            {},\n            \"Name\",\n        ),\n        (\n            \"create table t7 (id integer primary key, Title text);\",\n            \"t7\",\n            {},\n            \"Title\",\n        ),\n        # Two columns, one of which is id\n        (\n            \"create table t8 (id integer primary key, content text);\",\n            \"t8\",\n            {},\n            \"content\",\n        ),\n        (\n            \"create table t9 (pk integer primary key, content text);\",\n            \"t9\",\n            {},\n            \"content\",\n        ),\n    ],\n)\nasync def test_label_column_for_table(\n    create_sql, table_name, config, expected_label_column\n):\n    \"\"\"Test cases for label_column_for_table method\"\"\"\n    ds = Datasette()\n    db = ds.add_database(Database(ds, memory_name=\"test_label_column_for_table\"))\n    await db.execute_write_script(create_sql)\n    if config:\n        ds.config = {\"databases\": {\"test_label_column_for_table\": {\"tables\": config}}}\n    actual_label_column = await db.label_column_for_table(table_name)\n    if expected_label_column is None:\n        assert actual_label_column is None\n    else:\n        assert actual_label_column == expected_label_column\n"
  },
  {
    "path": "tests/test_load_extensions.py",
    "content": "from datasette.app import Datasette\nimport pytest\nfrom pathlib import Path\n\n# not necessarily a full path - the full compiled path looks like \"ext.dylib\"\n# or another suffix, but sqlite will, under the hood, decide which file\n# extension to use based on the operating system (apple=dylib, windows=dll etc)\n# this resolves to \"./ext\", which is enough for SQLite to calculate the rest\nCOMPILED_EXTENSION_PATH = str(Path(__file__).parent / \"ext\")\n\n\n# See if ext.c has been compiled, based off the different possible suffixes.\ndef has_compiled_ext():\n    for ext in [\"dylib\", \"so\", \"dll\"]:\n        path = Path(__file__).parent / f\"ext.{ext}\"\n        if path.is_file():\n            return True\n    return False\n\n\n@pytest.mark.asyncio\n@pytest.mark.skipif(not has_compiled_ext(), reason=\"Requires compiled ext.c\")\nasync def test_load_extension_default_entrypoint():\n    # The default entrypoint only loads a() and NOT b() or c(), so those\n    # should fail.\n    ds = Datasette(sqlite_extensions=[COMPILED_EXTENSION_PATH])\n\n    response = await ds.client.get(\"/_memory/-/query.json?_shape=arrays&sql=select+a()\")\n    assert response.status_code == 200\n    assert response.json()[\"rows\"][0][0] == \"a\"\n\n    response = await ds.client.get(\"/_memory/-/query.json?_shape=arrays&sql=select+b()\")\n    assert response.status_code == 400\n    assert response.json()[\"error\"] == \"no such function: b\"\n\n    response = await ds.client.get(\"/_memory/-/query.json?_shape=arrays&sql=select+c()\")\n    assert response.status_code == 400\n    assert response.json()[\"error\"] == \"no such function: c\"\n\n\n@pytest.mark.asyncio\n@pytest.mark.skipif(not has_compiled_ext(), reason=\"Requires compiled ext.c\")\nasync def test_load_extension_multiple_entrypoints():\n    # Load in the default entrypoint and the other 2 custom entrypoints, now\n    # all a(), b(), and c() should run successfully.\n    ds = Datasette(\n        sqlite_extensions=[\n            COMPILED_EXTENSION_PATH,\n            (COMPILED_EXTENSION_PATH, \"sqlite3_ext_b_init\"),\n            (COMPILED_EXTENSION_PATH, \"sqlite3_ext_c_init\"),\n        ]\n    )\n\n    response = await ds.client.get(\"/_memory/-/query.json?_shape=arrays&sql=select+a()\")\n    assert response.status_code == 200\n    assert response.json()[\"rows\"][0][0] == \"a\"\n\n    response = await ds.client.get(\"/_memory/-/query.json?_shape=arrays&sql=select+b()\")\n    assert response.status_code == 200\n    assert response.json()[\"rows\"][0][0] == \"b\"\n\n    response = await ds.client.get(\"/_memory/-/query.json?_shape=arrays&sql=select+c()\")\n    assert response.status_code == 200\n    assert response.json()[\"rows\"][0][0] == \"c\"\n"
  },
  {
    "path": "tests/test_messages.py",
    "content": "from .utils import cookie_was_deleted\nimport pytest\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"qs,expected\",\n    [\n        (\"add_msg=added-message\", [[\"added-message\", 1]]),\n        (\"add_msg=added-warning&type=WARNING\", [[\"added-warning\", 2]]),\n        (\"add_msg=added-error&type=ERROR\", [[\"added-error\", 3]]),\n    ],\n)\nasync def test_add_message_sets_cookie(ds_client, qs, expected):\n    response = await ds_client.get(f\"/fixtures/-/query.message?sql=select+1&{qs}\")\n    signed = response.cookies[\"ds_messages\"]\n    decoded = ds_client.ds.unsign(signed, \"messages\")\n    assert expected == decoded\n\n\n@pytest.mark.asyncio\nasync def test_messages_are_displayed_and_cleared(ds_client):\n    # First set the message cookie\n    set_msg_response = await ds_client.get(\n        \"/fixtures/-/query.message?sql=select+1&add_msg=xmessagex\"\n    )\n    # Now access a page that displays messages\n    response = await ds_client.get(\"/\", cookies=set_msg_response.cookies)\n    # Messages should be in that HTML\n    assert \"xmessagex\" in response.text\n    # Cookie should have been set that clears messages\n    assert cookie_was_deleted(response, \"ds_messages\")\n"
  },
  {
    "path": "tests/test_multipart.py",
    "content": "\"\"\"\nTests for request.form() multipart form data parsing.\n\nUses TDD approach - these tests are written first, then implementation follows.\n\"\"\"\n\nimport base64\nimport json\nimport pytest\nfrom collections import namedtuple\n\nfrom multipart_form_data_conformance import get_tests_dir\n\nfrom datasette.utils.asgi import Request, BadRequest\n\n\ndef make_receive(body: bytes):\n    \"\"\"Create an async receive callable that yields body in chunks.\"\"\"\n    consumed = False\n\n    async def receive():\n        nonlocal consumed\n        if consumed:\n            return {\"type\": \"http.request\", \"body\": b\"\", \"more_body\": False}\n        consumed = True\n        return {\"type\": \"http.request\", \"body\": body, \"more_body\": False}\n\n    return receive\n\n\ndef make_chunked_receive(body: bytes, chunk_size: int = 64):\n    \"\"\"Create an async receive callable that yields body in small chunks.\"\"\"\n    offset = 0\n\n    async def receive():\n        nonlocal offset\n        chunk = body[offset : offset + chunk_size]\n        offset += chunk_size\n        more_body = offset < len(body)\n        return {\"type\": \"http.request\", \"body\": chunk, \"more_body\": more_body}\n\n    return receive\n\n\ndef make_receive_with_noise(body: bytes):\n    \"\"\"\n    Create an async receive callable that includes an unexpected ASGI message.\n\n    The parser should ignore the unknown message type and continue.\n    \"\"\"\n    messages = [\n        {\"type\": \"http.response.start\", \"status\": 200, \"headers\": []},\n        {\"type\": \"http.request\", \"body\": body, \"more_body\": False},\n    ]\n    index = 0\n\n    async def receive():\n        nonlocal index\n        if index >= len(messages):\n            return {\"type\": \"http.request\", \"body\": b\"\", \"more_body\": False}\n        message = messages[index]\n        index += 1\n        return message\n\n    return receive\n\n\ndef make_disconnect_receive(body: bytes, chunk_size: int = 64):\n    \"\"\"\n    Create an async receive callable that disconnects mid-request.\n\n    The parser should raise on the disconnect.\n    \"\"\"\n    offset = 0\n    disconnected = False\n\n    async def receive():\n        nonlocal offset, disconnected\n        if disconnected:\n            return {\"type\": \"http.disconnect\"}\n        chunk = body[offset : offset + chunk_size]\n        offset += chunk_size\n        more_body = offset < len(body)\n        if more_body:\n            disconnected = True\n        return {\"type\": \"http.request\", \"body\": chunk, \"more_body\": more_body}\n\n    return receive\n\n\nclass TestFormUrlEncoded:\n    \"\"\"Test request.form() with application/x-www-form-urlencoded data.\"\"\"\n\n    @pytest.mark.asyncio\n    async def test_basic_form_fields(self):\n        \"\"\"Basic URL-encoded form should be parseable via request.form().\"\"\"\n        body = b\"username=john&password=secret\"\n        scope = {\n            \"type\": \"http\",\n            \"method\": \"POST\",\n            \"headers\": [\n                (b\"content-type\", b\"application/x-www-form-urlencoded\"),\n            ],\n        }\n        request = Request(scope, make_receive(body))\n\n        form = await request.form()\n\n        assert form[\"username\"] == \"john\"\n        assert form[\"password\"] == \"secret\"\n\n    @pytest.mark.asyncio\n    async def test_form_with_multiple_values(self):\n        \"\"\"Multiple values for same key should be accessible via getlist().\"\"\"\n        body = b\"tag=python&tag=web&tag=api\"\n        scope = {\n            \"type\": \"http\",\n            \"method\": \"POST\",\n            \"headers\": [\n                (b\"content-type\", b\"application/x-www-form-urlencoded\"),\n            ],\n        }\n        request = Request(scope, make_receive(body))\n\n        form = await request.form()\n\n        assert form[\"tag\"] == \"python\"  # First value\n        assert form.getlist(\"tag\") == [\"python\", \"web\", \"api\"]\n\n    @pytest.mark.asyncio\n    async def test_empty_form(self):\n        \"\"\"Empty form should return empty FormData.\"\"\"\n        body = b\"\"\n        scope = {\n            \"type\": \"http\",\n            \"method\": \"POST\",\n            \"headers\": [\n                (b\"content-type\", b\"application/x-www-form-urlencoded\"),\n            ],\n        }\n        request = Request(scope, make_receive(body))\n\n        form = await request.form()\n\n        assert len(form) == 0\n\n    @pytest.mark.asyncio\n    async def test_form_with_special_characters(self):\n        \"\"\"URL-encoded special characters should be decoded properly.\"\"\"\n        body = b\"message=hello%20world&emoji=%F0%9F%91%8B\"\n        scope = {\n            \"type\": \"http\",\n            \"method\": \"POST\",\n            \"headers\": [\n                (b\"content-type\", b\"application/x-www-form-urlencoded\"),\n            ],\n        }\n        request = Request(scope, make_receive(body))\n\n        form = await request.form()\n\n        assert form[\"message\"] == \"hello world\"\n        assert form[\"emoji\"] == \"👋\"\n\n\nclass TestMultipartBasic:\n    \"\"\"Test request.form() with multipart/form-data (fields only, no files).\"\"\"\n\n    @pytest.mark.asyncio\n    async def test_single_text_field(self):\n        \"\"\"Single text field in multipart should be parseable.\"\"\"\n        boundary = \"----TestBoundary123\"\n        body = (\n            b\"------TestBoundary123\\r\\n\"\n            b'Content-Disposition: form-data; name=\"username\"\\r\\n'\n            b\"\\r\\n\"\n            b\"john_doe\\r\\n\"\n            b\"------TestBoundary123--\\r\\n\"\n        )\n        scope = {\n            \"type\": \"http\",\n            \"method\": \"POST\",\n            \"headers\": [\n                (b\"content-type\", f\"multipart/form-data; boundary={boundary}\".encode()),\n            ],\n        }\n        request = Request(scope, make_receive(body))\n\n        form = await request.form()\n\n        assert form[\"username\"] == \"john_doe\"\n\n    @pytest.mark.asyncio\n    async def test_multiple_text_fields(self):\n        \"\"\"Multiple text fields in multipart should all be accessible.\"\"\"\n        boundary = \"----TestBoundary123\"\n        body = (\n            b\"------TestBoundary123\\r\\n\"\n            b'Content-Disposition: form-data; name=\"first_name\"\\r\\n'\n            b\"\\r\\n\"\n            b\"John\\r\\n\"\n            b\"------TestBoundary123\\r\\n\"\n            b'Content-Disposition: form-data; name=\"last_name\"\\r\\n'\n            b\"\\r\\n\"\n            b\"Doe\\r\\n\"\n            b\"------TestBoundary123--\\r\\n\"\n        )\n        scope = {\n            \"type\": \"http\",\n            \"method\": \"POST\",\n            \"headers\": [\n                (b\"content-type\", f\"multipart/form-data; boundary={boundary}\".encode()),\n            ],\n        }\n        request = Request(scope, make_receive(body))\n\n        form = await request.form()\n\n        assert form[\"first_name\"] == \"John\"\n        assert form[\"last_name\"] == \"Doe\"\n\n    @pytest.mark.asyncio\n    async def test_file_discarded_when_files_false(self):\n        \"\"\"File content should be discarded when files=False (default).\"\"\"\n        boundary = \"----TestBoundary123\"\n        body = (\n            b\"------TestBoundary123\\r\\n\"\n            b'Content-Disposition: form-data; name=\"title\"\\r\\n'\n            b\"\\r\\n\"\n            b\"My Document\\r\\n\"\n            b\"------TestBoundary123\\r\\n\"\n            b'Content-Disposition: form-data; name=\"file\"; filename=\"doc.txt\"\\r\\n'\n            b\"Content-Type: text/plain\\r\\n\"\n            b\"\\r\\n\"\n            b\"File content here\\r\\n\"\n            b\"------TestBoundary123\\r\\n\"\n            b'Content-Disposition: form-data; name=\"description\"\\r\\n'\n            b\"\\r\\n\"\n            b\"A sample document\\r\\n\"\n            b\"------TestBoundary123--\\r\\n\"\n        )\n        scope = {\n            \"type\": \"http\",\n            \"method\": \"POST\",\n            \"headers\": [\n                (b\"content-type\", f\"multipart/form-data; boundary={boundary}\".encode()),\n            ],\n        }\n        request = Request(scope, make_receive(body))\n\n        form = await request.form()  # files=False is default\n\n        # Text fields should be present\n        assert form[\"title\"] == \"My Document\"\n        assert form[\"description\"] == \"A sample document\"\n        # File should NOT be present\n        assert \"file\" not in form\n\n    @pytest.mark.asyncio\n    async def test_chunked_body_parsing(self):\n        \"\"\"Multipart should work when body arrives in small chunks.\"\"\"\n        boundary = \"----TestBoundary123\"\n        body = (\n            b\"------TestBoundary123\\r\\n\"\n            b'Content-Disposition: form-data; name=\"username\"\\r\\n'\n            b\"\\r\\n\"\n            b\"john_doe\\r\\n\"\n            b\"------TestBoundary123--\\r\\n\"\n        )\n        scope = {\n            \"type\": \"http\",\n            \"method\": \"POST\",\n            \"headers\": [\n                (b\"content-type\", f\"multipart/form-data; boundary={boundary}\".encode()),\n            ],\n        }\n        # Use small chunks to test streaming parser\n        request = Request(scope, make_chunked_receive(body, chunk_size=16))\n\n        form = await request.form()\n\n        assert form[\"username\"] == \"john_doe\"\n\n\nclass TestMultipartWithFiles:\n    \"\"\"Test request.form(files=True) for file uploads.\"\"\"\n\n    @pytest.mark.asyncio\n    async def test_single_file_upload(self):\n        \"\"\"Single file upload should create UploadedFile object.\"\"\"\n        boundary = \"----TestBoundary123\"\n        body = (\n            b\"------TestBoundary123\\r\\n\"\n            b'Content-Disposition: form-data; name=\"document\"; filename=\"test.txt\"\\r\\n'\n            b\"Content-Type: text/plain\\r\\n\"\n            b\"\\r\\n\"\n            b\"Hello, World!\\r\\n\"\n            b\"------TestBoundary123--\\r\\n\"\n        )\n        scope = {\n            \"type\": \"http\",\n            \"method\": \"POST\",\n            \"headers\": [\n                (b\"content-type\", f\"multipart/form-data; boundary={boundary}\".encode()),\n            ],\n        }\n        request = Request(scope, make_receive(body))\n\n        form = await request.form(files=True)\n\n        uploaded_file = form[\"document\"]\n        assert uploaded_file.filename == \"test.txt\"\n        assert uploaded_file.content_type == \"text/plain\"\n        assert await uploaded_file.read() == b\"Hello, World!\"\n        assert uploaded_file.size == 13\n\n    @pytest.mark.asyncio\n    async def test_mixed_fields_and_files(self):\n        \"\"\"Mixed form fields and files should all be accessible.\"\"\"\n        boundary = \"----TestBoundary123\"\n        body = (\n            b\"------TestBoundary123\\r\\n\"\n            b'Content-Disposition: form-data; name=\"title\"\\r\\n'\n            b\"\\r\\n\"\n            b\"My Document\\r\\n\"\n            b\"------TestBoundary123\\r\\n\"\n            b'Content-Disposition: form-data; name=\"file\"; filename=\"doc.txt\"\\r\\n'\n            b\"Content-Type: text/plain\\r\\n\"\n            b\"\\r\\n\"\n            b\"Document content\\r\\n\"\n            b\"------TestBoundary123\\r\\n\"\n            b'Content-Disposition: form-data; name=\"description\"\\r\\n'\n            b\"\\r\\n\"\n            b\"A sample\\r\\n\"\n            b\"------TestBoundary123--\\r\\n\"\n        )\n        scope = {\n            \"type\": \"http\",\n            \"method\": \"POST\",\n            \"headers\": [\n                (b\"content-type\", f\"multipart/form-data; boundary={boundary}\".encode()),\n            ],\n        }\n        request = Request(scope, make_receive(body))\n\n        form = await request.form(files=True)\n\n        # Text fields\n        assert form[\"title\"] == \"My Document\"\n        assert form[\"description\"] == \"A sample\"\n        # File\n        uploaded_file = form[\"file\"]\n        assert uploaded_file.filename == \"doc.txt\"\n        assert await uploaded_file.read() == b\"Document content\"\n\n    @pytest.mark.asyncio\n    async def test_multiple_files_same_name(self):\n        \"\"\"Multiple files with same name should be accessible via getlist().\"\"\"\n        boundary = \"----TestBoundary123\"\n        body = (\n            b\"------TestBoundary123\\r\\n\"\n            b'Content-Disposition: form-data; name=\"files\"; filename=\"a.txt\"\\r\\n'\n            b\"Content-Type: text/plain\\r\\n\"\n            b\"\\r\\n\"\n            b\"File A\\r\\n\"\n            b\"------TestBoundary123\\r\\n\"\n            b'Content-Disposition: form-data; name=\"files\"; filename=\"b.txt\"\\r\\n'\n            b\"Content-Type: text/plain\\r\\n\"\n            b\"\\r\\n\"\n            b\"File B\\r\\n\"\n            b\"------TestBoundary123--\\r\\n\"\n        )\n        scope = {\n            \"type\": \"http\",\n            \"method\": \"POST\",\n            \"headers\": [\n                (b\"content-type\", f\"multipart/form-data; boundary={boundary}\".encode()),\n            ],\n        }\n        request = Request(scope, make_receive(body))\n\n        form = await request.form(files=True)\n\n        files = form.getlist(\"files\")\n        assert len(files) == 2\n        assert files[0].filename == \"a.txt\"\n        assert files[1].filename == \"b.txt\"\n\n    @pytest.mark.asyncio\n    async def test_large_file_spills_to_disk(self):\n        \"\"\"Files larger than threshold should spill to temp file.\"\"\"\n        boundary = \"----TestBoundary123\"\n        # Create a body larger than the in-memory threshold (1MB)\n        large_content = b\"x\" * (2 * 1024 * 1024)  # 2MB\n        body = (\n            b\"------TestBoundary123\\r\\n\"\n            b'Content-Disposition: form-data; name=\"bigfile\"; filename=\"large.bin\"\\r\\n'\n            b\"Content-Type: application/octet-stream\\r\\n\"\n            b\"\\r\\n\" + large_content + b\"\\r\\n\"\n            b\"------TestBoundary123--\\r\\n\"\n        )\n        scope = {\n            \"type\": \"http\",\n            \"method\": \"POST\",\n            \"headers\": [\n                (b\"content-type\", f\"multipart/form-data; boundary={boundary}\".encode()),\n            ],\n        }\n        request = Request(scope, make_receive(body))\n\n        form = await request.form(files=True)\n\n        uploaded_file = form[\"bigfile\"]\n        assert uploaded_file.size == len(large_content)\n        # Content should still be readable\n        content = await uploaded_file.read()\n        assert content == large_content\n\n    @pytest.mark.asyncio\n    async def test_uploaded_file_seek_and_read(self):\n        \"\"\"UploadedFile should support seek and multiple reads.\"\"\"\n        boundary = \"----TestBoundary123\"\n        body = (\n            b\"------TestBoundary123\\r\\n\"\n            b'Content-Disposition: form-data; name=\"file\"; filename=\"test.txt\"\\r\\n'\n            b\"Content-Type: text/plain\\r\\n\"\n            b\"\\r\\n\"\n            b\"Hello, World!\\r\\n\"\n            b\"------TestBoundary123--\\r\\n\"\n        )\n        scope = {\n            \"type\": \"http\",\n            \"method\": \"POST\",\n            \"headers\": [\n                (b\"content-type\", f\"multipart/form-data; boundary={boundary}\".encode()),\n            ],\n        }\n        request = Request(scope, make_receive(body))\n\n        form = await request.form(files=True)\n        uploaded_file = form[\"file\"]\n\n        # First read\n        content1 = await uploaded_file.read()\n        assert content1 == b\"Hello, World!\"\n\n        # Seek back to start\n        await uploaded_file.seek(0)\n\n        # Second read\n        content2 = await uploaded_file.read()\n        assert content2 == b\"Hello, World!\"\n\n\nclass TestMultipartCleanup:\n    \"\"\"Test deterministic cleanup of uploaded files.\"\"\"\n\n    @pytest.mark.asyncio\n    async def test_formdata_close_closes_uploaded_files(self):\n        boundary = \"----TestBoundary123\"\n        body = (\n            b\"------TestBoundary123\\r\\n\"\n            b'Content-Disposition: form-data; name=\"file\"; filename=\"test.txt\"\\r\\n'\n            b\"Content-Type: text/plain\\r\\n\"\n            b\"\\r\\n\"\n            b\"Hello\\r\\n\"\n            b\"------TestBoundary123--\\r\\n\"\n        )\n        scope = {\n            \"type\": \"http\",\n            \"method\": \"POST\",\n            \"headers\": [\n                (b\"content-type\", f\"multipart/form-data; boundary={boundary}\".encode()),\n            ],\n        }\n        request = Request(scope, make_receive(body))\n        form = await request.form(files=True)\n        uploaded_file = form[\"file\"]\n\n        form.close()\n\n        with pytest.raises(ValueError):\n            await uploaded_file.read()\n\n    @pytest.mark.asyncio\n    async def test_formdata_async_context_manager_closes_files(self):\n        boundary = \"----TestBoundary123\"\n        body = (\n            b\"------TestBoundary123\\r\\n\"\n            b'Content-Disposition: form-data; name=\"file\"; filename=\"test.txt\"\\r\\n'\n            b\"Content-Type: text/plain\\r\\n\"\n            b\"\\r\\n\"\n            b\"Hello\\r\\n\"\n            b\"------TestBoundary123--\\r\\n\"\n        )\n        scope = {\n            \"type\": \"http\",\n            \"method\": \"POST\",\n            \"headers\": [\n                (b\"content-type\", f\"multipart/form-data; boundary={boundary}\".encode()),\n            ],\n        }\n        request = Request(scope, make_receive(body))\n        form = await request.form(files=True)\n        uploaded_file = form[\"file\"]\n\n        async with form:\n            pass\n\n        with pytest.raises(ValueError):\n            await uploaded_file.read()\n\n\nclass TestMultipartEdgeCases:\n    \"\"\"Test edge cases in multipart parsing.\"\"\"\n\n    @pytest.mark.asyncio\n    async def test_empty_file_upload(self):\n        \"\"\"Empty file (filename but no content) should be handled.\"\"\"\n        boundary = \"----TestBoundary123\"\n        body = (\n            b\"------TestBoundary123\\r\\n\"\n            b'Content-Disposition: form-data; name=\"file\"; filename=\"empty.txt\"\\r\\n'\n            b\"Content-Type: text/plain\\r\\n\"\n            b\"\\r\\n\"\n            b\"\\r\\n\"\n            b\"------TestBoundary123--\\r\\n\"\n        )\n        scope = {\n            \"type\": \"http\",\n            \"method\": \"POST\",\n            \"headers\": [\n                (b\"content-type\", f\"multipart/form-data; boundary={boundary}\".encode()),\n            ],\n        }\n        request = Request(scope, make_receive(body))\n\n        form = await request.form(files=True)\n\n        uploaded_file = form[\"file\"]\n        assert uploaded_file.filename == \"empty.txt\"\n        assert uploaded_file.size == 0\n        assert await uploaded_file.read() == b\"\"\n\n    @pytest.mark.asyncio\n    async def test_filename_with_path(self):\n        \"\"\"Filename containing path should extract just the filename.\"\"\"\n        boundary = \"----TestBoundary123\"\n        body = (\n            b\"------TestBoundary123\\r\\n\"\n            b'Content-Disposition: form-data; name=\"file\"; filename=\"C:\\\\Users\\\\test\\\\doc.txt\"\\r\\n'\n            b\"Content-Type: text/plain\\r\\n\"\n            b\"\\r\\n\"\n            b\"content\\r\\n\"\n            b\"------TestBoundary123--\\r\\n\"\n        )\n        scope = {\n            \"type\": \"http\",\n            \"method\": \"POST\",\n            \"headers\": [\n                (b\"content-type\", f\"multipart/form-data; boundary={boundary}\".encode()),\n            ],\n        }\n        request = Request(scope, make_receive(body))\n\n        form = await request.form(files=True)\n\n        # Should extract just the filename, not the full path\n        uploaded_file = form[\"file\"]\n        assert uploaded_file.filename == \"doc.txt\"\n\n    @pytest.mark.asyncio\n    async def test_missing_content_type_header(self):\n        \"\"\"Missing content-type in request should raise BadRequest.\"\"\"\n        body = b\"some body\"\n        scope = {\n            \"type\": \"http\",\n            \"method\": \"POST\",\n            \"headers\": [],\n        }\n        request = Request(scope, make_receive(body))\n\n        with pytest.raises(BadRequest):\n            await request.form()\n\n    @pytest.mark.asyncio\n    async def test_invalid_content_type(self):\n        \"\"\"Non-form content-type should raise BadRequest.\"\"\"\n        body = b'{\"key\": \"value\"}'\n        scope = {\n            \"type\": \"http\",\n            \"method\": \"POST\",\n            \"headers\": [\n                (b\"content-type\", b\"application/json\"),\n            ],\n        }\n        request = Request(scope, make_receive(body))\n\n        with pytest.raises(BadRequest):\n            await request.form()\n\n    @pytest.mark.asyncio\n    async def test_missing_boundary(self):\n        \"\"\"Multipart without boundary should raise BadRequest.\"\"\"\n        body = b\"some body\"\n        scope = {\n            \"type\": \"http\",\n            \"method\": \"POST\",\n            \"headers\": [\n                (b\"content-type\", b\"multipart/form-data\"),\n            ],\n        }\n        request = Request(scope, make_receive(body))\n\n        with pytest.raises(BadRequest):\n            await request.form()\n\n\nclass TestSecurityLimits:\n    \"\"\"Test security limits on form parsing.\"\"\"\n\n    @pytest.mark.asyncio\n    async def test_max_fields_limit(self):\n        \"\"\"Should reject requests with too many fields.\"\"\"\n        boundary = \"----TestBoundary123\"\n        # Create body with many fields\n        parts = []\n        for i in range(1001):  # Default max is 1000\n            parts.append(\n                f\"------TestBoundary123\\r\\n\"\n                f'Content-Disposition: form-data; name=\"field{i}\"\\r\\n'\n                f\"\\r\\n\"\n                f\"value{i}\\r\\n\"\n            )\n        parts.append(\"------TestBoundary123--\\r\\n\")\n        body = \"\".join(parts).encode()\n\n        scope = {\n            \"type\": \"http\",\n            \"method\": \"POST\",\n            \"headers\": [\n                (b\"content-type\", f\"multipart/form-data; boundary={boundary}\".encode()),\n            ],\n        }\n        request = Request(scope, make_receive(body))\n\n        with pytest.raises(BadRequest, match=\"(?i)too many\"):\n            await request.form(max_fields=1000)\n\n    @pytest.mark.asyncio\n    async def test_max_file_size_limit(self):\n        \"\"\"Should reject files exceeding size limit.\"\"\"\n        boundary = \"----TestBoundary123\"\n        large_content = b\"x\" * (11 * 1024 * 1024)  # 11MB\n        body = (\n            b\"------TestBoundary123\\r\\n\"\n            b'Content-Disposition: form-data; name=\"file\"; filename=\"big.bin\"\\r\\n'\n            b\"Content-Type: application/octet-stream\\r\\n\"\n            b\"\\r\\n\" + large_content + b\"\\r\\n\"\n            b\"------TestBoundary123--\\r\\n\"\n        )\n        scope = {\n            \"type\": \"http\",\n            \"method\": \"POST\",\n            \"headers\": [\n                (b\"content-type\", f\"multipart/form-data; boundary={boundary}\".encode()),\n            ],\n        }\n        request = Request(scope, make_receive(body))\n\n        with pytest.raises(BadRequest, match=\"(?i)file.*too large|too large\"):\n            await request.form(files=True, max_file_size=10 * 1024 * 1024)\n\n    @pytest.mark.asyncio\n    async def test_max_request_size_limit(self):\n        \"\"\"Should reject requests exceeding total size limit.\"\"\"\n        boundary = \"----TestBoundary123\"\n        large_content = b\"x\" * (6 * 1024 * 1024)  # 6MB\n        body = (\n            b\"------TestBoundary123\\r\\n\"\n            b'Content-Disposition: form-data; name=\"file\"; filename=\"big.bin\"\\r\\n'\n            b\"Content-Type: application/octet-stream\\r\\n\"\n            b\"\\r\\n\" + large_content + b\"\\r\\n\"\n            b\"------TestBoundary123--\\r\\n\"\n        )\n        scope = {\n            \"type\": \"http\",\n            \"method\": \"POST\",\n            \"headers\": [\n                (b\"content-type\", f\"multipart/form-data; boundary={boundary}\".encode()),\n            ],\n        }\n        request = Request(scope, make_receive(body))\n\n        with pytest.raises(BadRequest, match=\"(?i)too large|request.*too large\"):\n            await request.form(files=True, max_request_size=5 * 1024 * 1024)\n\n\nclass TestMultipartStrictnessAndLimits:\n    \"\"\"Tests that enforce stricter ASGI and multipart behaviors.\"\"\"\n\n    @pytest.mark.asyncio\n    async def test_multipart_truncated_body_is_error(self):\n        \"\"\"Truncated multipart without closing boundary should raise.\"\"\"\n        boundary = \"----TestBoundary123\"\n        # Missing the final closing boundary line\n        body = (\n            b\"------TestBoundary123\\r\\n\"\n            b'Content-Disposition: form-data; name=\"field\"\\r\\n'\n            b\"\\r\\n\"\n            b\"value\\r\\n\"\n        )\n        scope = {\n            \"type\": \"http\",\n            \"method\": \"POST\",\n            \"headers\": [\n                (b\"content-type\", f\"multipart/form-data; boundary={boundary}\".encode()),\n            ],\n        }\n        request = Request(scope, make_receive(body))\n\n        with pytest.raises(BadRequest, match=\"Truncated multipart body\"):\n            await request.form()\n\n    @pytest.mark.asyncio\n    async def test_disconnect_mid_body_is_error(self):\n        \"\"\"Client disconnect during body streaming should raise.\"\"\"\n        boundary = \"----TestBoundary123\"\n        body = (\n            b\"------TestBoundary123\\r\\n\"\n            b'Content-Disposition: form-data; name=\"field\"\\r\\n'\n            b\"\\r\\n\"\n            b\"value\\r\\n\"\n            b\"------TestBoundary123--\\r\\n\"\n        )\n        scope = {\n            \"type\": \"http\",\n            \"method\": \"POST\",\n            \"headers\": [\n                (b\"content-type\", f\"multipart/form-data; boundary={boundary}\".encode()),\n            ],\n        }\n        request = Request(scope, make_disconnect_receive(body, chunk_size=16))\n\n        with pytest.raises(BadRequest, match=\"disconnected\"):\n            await request.form()\n\n    @pytest.mark.asyncio\n    async def test_unknown_asgi_message_type_is_ignored(self):\n        \"\"\"Unexpected ASGI message types should be ignored.\"\"\"\n        boundary = \"----TestBoundary123\"\n        body = (\n            b\"------TestBoundary123\\r\\n\"\n            b'Content-Disposition: form-data; name=\"field\"\\r\\n'\n            b\"\\r\\n\"\n            b\"value\\r\\n\"\n            b\"------TestBoundary123--\\r\\n\"\n        )\n        scope = {\n            \"type\": \"http\",\n            \"method\": \"POST\",\n            \"headers\": [\n                (b\"content-type\", f\"multipart/form-data; boundary={boundary}\".encode()),\n            ],\n        }\n        request = Request(scope, make_receive_with_noise(body))\n\n        form = await request.form()\n        assert form[\"field\"] == \"value\"\n\n    @pytest.mark.asyncio\n    async def test_max_files_enforced_even_when_files_false(self):\n        \"\"\"File count limits should apply even when file handling is disabled.\"\"\"\n        boundary = \"----TestBoundary123\"\n        body = (\n            b\"------TestBoundary123\\r\\n\"\n            b'Content-Disposition: form-data; name=\"f1\"; filename=\"a.txt\"\\r\\n'\n            b\"Content-Type: text/plain\\r\\n\"\n            b\"\\r\\n\"\n            b\"a\\r\\n\"\n            b\"------TestBoundary123\\r\\n\"\n            b'Content-Disposition: form-data; name=\"f2\"; filename=\"b.txt\"\\r\\n'\n            b\"Content-Type: text/plain\\r\\n\"\n            b\"\\r\\n\"\n            b\"b\\r\\n\"\n            b\"------TestBoundary123--\\r\\n\"\n        )\n        scope = {\n            \"type\": \"http\",\n            \"method\": \"POST\",\n            \"headers\": [\n                (b\"content-type\", f\"multipart/form-data; boundary={boundary}\".encode()),\n            ],\n        }\n        request = Request(scope, make_receive(body))\n\n        with pytest.raises(BadRequest, match=\"Too many files\"):\n            await request.form(files=False, max_files=1)\n\n    @pytest.mark.asyncio\n    async def test_max_parts_limit(self):\n        \"\"\"Total part count should be bounded.\"\"\"\n        boundary = \"----TestBoundary123\"\n        body = (\n            b\"------TestBoundary123\\r\\n\"\n            b'Content-Disposition: form-data; name=\"a\"\\r\\n'\n            b\"\\r\\n\"\n            b\"1\\r\\n\"\n            b\"------TestBoundary123\\r\\n\"\n            b'Content-Disposition: form-data; name=\"b\"\\r\\n'\n            b\"\\r\\n\"\n            b\"2\\r\\n\"\n            b\"------TestBoundary123--\\r\\n\"\n        )\n        scope = {\n            \"type\": \"http\",\n            \"method\": \"POST\",\n            \"headers\": [\n                (b\"content-type\", f\"multipart/form-data; boundary={boundary}\".encode()),\n            ],\n        }\n        request = Request(scope, make_receive(body))\n\n        with pytest.raises(BadRequest, match=\"Too many parts\"):\n            await request.form(max_parts=1)\n\n    @pytest.mark.asyncio\n    async def test_max_file_size_enforced_even_when_files_false(self):\n        \"\"\"File size limits should apply even when file handling is disabled.\"\"\"\n        boundary = \"----TestBoundary123\"\n        big_content = b\"x\" * 2048\n        body = (\n            b\"------TestBoundary123\\r\\n\"\n            b'Content-Disposition: form-data; name=\"file\"; filename=\"big.bin\"\\r\\n'\n            b\"Content-Type: application/octet-stream\\r\\n\"\n            b\"\\r\\n\" + big_content + b\"\\r\\n\"\n            b\"------TestBoundary123--\\r\\n\"\n        )\n        scope = {\n            \"type\": \"http\",\n            \"method\": \"POST\",\n            \"headers\": [\n                (b\"content-type\", f\"multipart/form-data; boundary={boundary}\".encode()),\n            ],\n        }\n        request = Request(scope, make_receive(body))\n\n        with pytest.raises(BadRequest, match=\"File too large\"):\n            await request.form(files=False, max_file_size=1024)\n\n    @pytest.mark.asyncio\n    async def test_part_header_limits(self):\n        \"\"\"Overly large part headers should be rejected.\"\"\"\n        boundary = \"----TestBoundary123\"\n        huge_header_value = \"x\" * 5000\n        body = (\n            b\"------TestBoundary123\\r\\n\"\n            + f'Content-Disposition: form-data; name=\"field\"; foo=\"{huge_header_value}\"\\r\\n'.encode()\n            + b\"\\r\\n\"\n            + b\"value\\r\\n\"\n            + b\"------TestBoundary123--\\r\\n\"\n        )\n        scope = {\n            \"type\": \"http\",\n            \"method\": \"POST\",\n            \"headers\": [\n                (b\"content-type\", f\"multipart/form-data; boundary={boundary}\".encode()),\n            ],\n        }\n        request = Request(scope, make_receive(body))\n\n        with pytest.raises(BadRequest, match=\"headers too large\"):\n            await request.form(max_part_header_bytes=1024)\n\n    @pytest.mark.asyncio\n    async def test_insufficient_disk_space_rejects_upload(self, monkeypatch):\n        \"\"\"Uploads should be rejected when free disk is below the floor.\"\"\"\n        boundary = \"----TestBoundary123\"\n        body = (\n            b\"------TestBoundary123\\r\\n\"\n            b'Content-Disposition: form-data; name=\"file\"; filename=\"test.txt\"\\r\\n'\n            b\"Content-Type: text/plain\\r\\n\"\n            b\"\\r\\n\"\n            b\"Hello\\r\\n\"\n            b\"------TestBoundary123--\\r\\n\"\n        )\n        scope = {\n            \"type\": \"http\",\n            \"method\": \"POST\",\n            \"headers\": [\n                (b\"content-type\", f\"multipart/form-data; boundary={boundary}\".encode()),\n            ],\n        }\n\n        DiskUsage = namedtuple(\"DiskUsage\", (\"total\", \"used\", \"free\"))\n        monkeypatch.setattr(\n            \"datasette.utils.multipart.shutil.disk_usage\",\n            lambda path: DiskUsage(total=100, used=95, free=5),\n        )\n\n        request = Request(scope, make_receive(body))\n        with pytest.raises(BadRequest, match=\"Insufficient disk space\"):\n            await request.form(files=True, min_free_disk_bytes=50)\n\n    @pytest.mark.asyncio\n    async def test_low_disk_space_does_not_block_field_only_forms(self, monkeypatch):\n        \"\"\"Low disk space should not reject multipart forms with no file parts.\"\"\"\n        boundary = \"----TestBoundary123\"\n        body = (\n            b\"------TestBoundary123\\r\\n\"\n            b'Content-Disposition: form-data; name=\"field\"\\r\\n'\n            b\"\\r\\n\"\n            b\"value\\r\\n\"\n            b\"------TestBoundary123--\\r\\n\"\n        )\n        scope = {\n            \"type\": \"http\",\n            \"method\": \"POST\",\n            \"headers\": [\n                (b\"content-type\", f\"multipart/form-data; boundary={boundary}\".encode()),\n            ],\n        }\n\n        DiskUsage = namedtuple(\"DiskUsage\", (\"total\", \"used\", \"free\"))\n        monkeypatch.setattr(\n            \"datasette.utils.multipart.shutil.disk_usage\",\n            lambda path: DiskUsage(total=100, used=99, free=1),\n        )\n\n        request = Request(scope, make_receive(body))\n        form = await request.form(files=True, min_free_disk_bytes=50)\n        assert form[\"field\"] == \"value\"\n\n    @pytest.mark.asyncio\n    async def test_headers_without_newline_hit_header_byte_limit(self):\n        \"\"\"Headers that never terminate should still hit the header byte limit.\"\"\"\n        boundary = \"----TestBoundary123\"\n        huge = b\"x\" * 5000\n        # No CRLF is included after the header line\n        body = (\n            b\"------TestBoundary123\\r\\n\"\n            b'Content-Disposition: form-data; name=\"field\"; foo=\"' + huge + b'\"'\n        )\n        scope = {\n            \"type\": \"http\",\n            \"method\": \"POST\",\n            \"headers\": [\n                (b\"content-type\", f\"multipart/form-data; boundary={boundary}\".encode()),\n            ],\n        }\n        request = Request(scope, make_receive(body))\n\n        with pytest.raises(BadRequest, match=\"headers too large\"):\n            await request.form(max_part_header_bytes=1024)\n\n\nclass TestFormDataLenSemantics:\n    \"\"\"Test that FormData.__len__ reflects number of items, not unique keys.\"\"\"\n\n    @pytest.mark.asyncio\n    async def test_len_counts_items(self):\n        body = b\"tag=python&tag=web&tag=api\"\n        scope = {\n            \"type\": \"http\",\n            \"method\": \"POST\",\n            \"headers\": [\n                (b\"content-type\", b\"application/x-www-form-urlencoded\"),\n            ],\n        }\n        request = Request(scope, make_receive(body))\n\n        form = await request.form()\n        assert len(form) == 3\n\n\n# Conformance test suite using multipart-form-data-conformance\n\n# Tests where our parser intentionally differs from strict spec for security/practicality\n# Our parser sanitizes filenames (strips paths) while the conformance suite expects raw\nFILENAME_SANITIZATION_TESTS = {\n    \"026-filename-with-backslash\",  # We preserve backslashes but they test expects raw\n    \"029-filename-path-traversal\",  # We strip path components for security\n}\n\n# Tests for optional/lenient features we don't implement\nOPTIONAL_TESTS = {\n    \"085-header-folding\",  # Obsolete header folding feature\n}\n\n# Tests for malformed input where we're lenient instead of erroring\nLENIENT_PARSING_TESTS = {\n    \"203-missing-content-disposition\",\n    \"204-invalid-content-disposition\",\n}\n\n\ndef load_conformance_test_cases():\n    \"\"\"Load all test cases from multipart-form-data-conformance.\"\"\"\n    tests_dir = get_tests_dir()\n    test_cases = []\n\n    for category_dir in sorted(tests_dir.iterdir()):\n        if not category_dir.is_dir():\n            continue\n        for test_dir in sorted(category_dir.iterdir()):\n            if not test_dir.is_dir():\n                continue\n            test_json = test_dir / \"test.json\"\n            headers_json = test_dir / \"headers.json\"\n            input_raw = test_dir / \"input.raw\"\n\n            if not all(f.exists() for f in [test_json, headers_json, input_raw]):\n                continue\n\n            with open(test_json) as f:\n                test_spec = json.load(f)\n            with open(headers_json) as f:\n                headers = json.load(f)\n            with open(input_raw, \"rb\") as f:\n                body = f.read()\n\n            test_id = test_spec[\"id\"]\n\n            # Add marks for tests we handle differently\n            marks = []\n            if test_id in FILENAME_SANITIZATION_TESTS:\n                marks.append(\n                    pytest.mark.xfail(reason=\"Parser sanitizes filenames for security\")\n                )\n            elif test_id in OPTIONAL_TESTS:\n                marks.append(\n                    pytest.mark.xfail(reason=\"Optional feature not implemented\")\n                )\n            elif test_id in LENIENT_PARSING_TESTS:\n                marks.append(\n                    pytest.mark.xfail(reason=\"Parser is lenient with malformed input\")\n                )\n\n            test_cases.append(\n                pytest.param(\n                    test_spec,\n                    headers,\n                    body,\n                    id=test_id,\n                    marks=marks,\n                )\n            )\n\n    return test_cases\n\n\nCONFORMANCE_TEST_CASES = load_conformance_test_cases()\n\n\n@pytest.mark.parametrize(\"test_spec,headers,body\", CONFORMANCE_TEST_CASES)\n@pytest.mark.asyncio\nasync def test_conformance(test_spec, headers, body):\n    \"\"\"\n    Run conformance test cases from multipart-form-data-conformance.\n\n    Each test case specifies:\n    - headers: HTTP headers including Content-Type with boundary\n    - body: Raw multipart body bytes\n    - expected: Expected parse result (valid/invalid, parts list)\n    \"\"\"\n    scope = {\n        \"type\": \"http\",\n        \"method\": \"POST\",\n        \"headers\": [(k.encode(), v.encode()) for k, v in headers.items()],\n    }\n    request = Request(scope, make_receive(body))\n\n    expected = test_spec[\"expected\"]\n\n    if not expected[\"valid\"]:\n        # Should raise an error for invalid input\n        with pytest.raises((BadRequest, ValueError)):\n            await request.form(files=True)\n        return\n\n    # Parse form data\n    form = await request.form(files=True)\n\n    # Verify each expected part\n    for i, expected_part in enumerate(expected[\"parts\"]):\n        name = expected_part[\"name\"]\n\n        # Get value(s) for this name\n        values = form.getlist(name)\n\n        # Find the value at the correct index for this name\n        # (handles multiple values with same name)\n        same_name_count = sum(1 for p in expected[\"parts\"][:i] if p[\"name\"] == name)\n\n        if same_name_count >= len(values):\n            pytest.fail(\n                f\"Expected part {name} at index {same_name_count} but only {len(values)} found\"\n            )\n\n        value = values[same_name_count]\n\n        # Determine expected content\n        if \"body_base64\" in expected_part:\n            expected_content = base64.b64decode(expected_part[\"body_base64\"])\n        elif \"body_text\" in expected_part:\n            expected_content = expected_part[\"body_text\"].encode(\"utf-8\")\n        else:\n            expected_content = None\n\n        # Check for file vs field\n        # A part is a file if it has a filename OR filename_star\n        is_file = (\n            expected_part.get(\"filename\") is not None\n            or expected_part.get(\"filename_star\") is not None\n        )\n\n        if is_file:\n            # It's a file\n            assert hasattr(value, \"filename\"), f\"Expected file for {name}\"\n\n            # Check filename - use filename_star if present, else filename\n            expected_filename = expected_part.get(\"filename_star\") or expected_part.get(\n                \"filename\"\n            )\n            if expected_filename:\n                assert (\n                    value.filename == expected_filename\n                ), f\"Filename mismatch: expected {expected_filename!r}, got {value.filename!r}\"\n\n            if expected_part.get(\"content_type\"):\n                assert value.content_type == expected_part[\"content_type\"]\n\n            content = await value.read()\n            assert (\n                len(content) == expected_part[\"body_size\"]\n            ), f\"Size mismatch: expected {expected_part['body_size']}, got {len(content)}\"\n            if expected_content is not None:\n                assert content == expected_content\n        else:\n            # It's a text field\n            if hasattr(value, \"filename\"):\n                pytest.fail(f\"Expected text field for {name}, got file\")\n\n            if expected_content is not None:\n                # For text fields, value is a string\n                try:\n                    expected_text = expected_content.decode(\"utf-8\")\n                except UnicodeDecodeError:\n                    expected_text = expected_content.decode(\"latin-1\")\n                assert (\n                    value == expected_text\n                ), f\"Value mismatch: expected {expected_text!r}, got {value!r}\"\n"
  },
  {
    "path": "tests/test_package.py",
    "content": "from click.testing import CliRunner\nfrom datasette import cli\nfrom unittest import mock\nimport os\nimport pathlib\nimport pytest\n\n\nclass CaptureDockerfile:\n    def __call__(self, _):\n        self.captured = (pathlib.Path() / \"Dockerfile\").read_text()\n\n\nEXPECTED_DOCKERFILE = \"\"\"\nFROM python:3.11.0-slim-bullseye\nCOPY . /app\nWORKDIR /app\n\nENV DATASETTE_SECRET 'sekrit'\nRUN pip install -U datasette\nRUN datasette inspect test.db --inspect-file inspect-data.json\nENV PORT {port}\nEXPOSE {port}\nCMD datasette serve --host 0.0.0.0 -i test.db --cors --inspect-file inspect-data.json --port $PORT\n\"\"\".strip()\n\n\n@pytest.mark.serial\n@mock.patch(\"shutil.which\")\n@mock.patch(\"datasette.cli.call\")\ndef test_package(mock_call, mock_which, tmp_path_factory):\n    mock_which.return_value = True\n    runner = CliRunner()\n    capture = CaptureDockerfile()\n    mock_call.side_effect = capture\n    os.chdir(tmp_path_factory.mktemp(\"runner\"))\n    with open(\"test.db\", \"w\") as fp:\n        fp.write(\"data\")\n    result = runner.invoke(cli.cli, [\"package\", \"test.db\", \"--secret\", \"sekrit\"])\n    assert 0 == result.exit_code\n    mock_call.assert_has_calls([mock.call([\"docker\", \"build\", \".\"])])\n    assert EXPECTED_DOCKERFILE.format(port=8001) == capture.captured\n\n\n@mock.patch(\"shutil.which\")\n@mock.patch(\"datasette.cli.call\")\ndef test_package_with_port(mock_call, mock_which, tmp_path_factory):\n    mock_which.return_value = True\n    capture = CaptureDockerfile()\n    mock_call.side_effect = capture\n    runner = CliRunner()\n    os.chdir(tmp_path_factory.mktemp(\"runner\"))\n    with open(\"test.db\", \"w\") as fp:\n        fp.write(\"data\")\n    result = runner.invoke(\n        cli.cli, [\"package\", \"test.db\", \"-p\", \"8080\", \"--secret\", \"sekrit\"]\n    )\n    assert 0 == result.exit_code\n    assert EXPECTED_DOCKERFILE.format(port=8080) == capture.captured\n"
  },
  {
    "path": "tests/test_permission_endpoints.py",
    "content": "\"\"\"\nTests for permission endpoints:\n- /-/allowed.json\n- /-/rules.json\n\"\"\"\n\nimport pytest\nimport pytest_asyncio\nfrom datasette.app import Datasette\n\n\n@pytest_asyncio.fixture\nasync def ds_with_permissions():\n    \"\"\"Create a Datasette instance with test data and permissions.\"\"\"\n    ds = Datasette()\n    ds.root_enabled = True\n    await ds.invoke_startup()\n\n    # Add some test databases and tables\n    db = ds.add_memory_database(\"analytics\")\n    await db.execute_write(\n        \"CREATE TABLE IF NOT EXISTS users (id INTEGER PRIMARY KEY, name TEXT, email TEXT)\"\n    )\n    await db.execute_write(\n        \"CREATE TABLE IF NOT EXISTS events (id INTEGER PRIMARY KEY, event_type TEXT, user_id INTEGER)\"\n    )\n\n    db2 = ds.add_memory_database(\"production\")\n    await db2.execute_write(\n        \"CREATE TABLE IF NOT EXISTS orders (id INTEGER PRIMARY KEY, total REAL)\"\n    )\n    await db2.execute_write(\n        \"CREATE TABLE IF NOT EXISTS customers (id INTEGER PRIMARY KEY, name TEXT)\"\n    )\n\n    await ds.refresh_schemas()\n\n    return ds\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"path,expected_status,expected_keys\",\n    [\n        # Instance level permission\n        (\n            \"/-/allowed.json?action=view-instance\",\n            200,\n            {\"action\", \"items\", \"total\", \"page\"},\n        ),\n        # Database level permission\n        (\n            \"/-/allowed.json?action=view-database\",\n            200,\n            {\"action\", \"items\", \"total\", \"page\"},\n        ),\n        # Table level permission\n        (\n            \"/-/allowed.json?action=view-table\",\n            200,\n            {\"action\", \"items\", \"total\", \"page\"},\n        ),\n        (\n            \"/-/allowed.json?action=execute-sql\",\n            200,\n            {\"action\", \"items\", \"total\", \"page\"},\n        ),\n        # Missing action parameter\n        (\"/-/allowed.json\", 400, {\"error\"}),\n        # Invalid action\n        (\"/-/allowed.json?action=nonexistent\", 404, {\"error\"}),\n        # Any valid action works, even if no permission rules exist for it\n        (\n            \"/-/allowed.json?action=insert-row\",\n            200,\n            {\"action\", \"items\", \"total\", \"page\"},\n        ),\n    ],\n)\nasync def test_allowed_json_basic(\n    ds_with_permissions, path, expected_status, expected_keys\n):\n    response = await ds_with_permissions.client.get(path)\n    assert response.status_code == expected_status\n    data = response.json()\n    assert expected_keys.issubset(data.keys())\n\n\n@pytest.mark.asyncio\nasync def test_allowed_json_response_structure(ds_with_permissions):\n    \"\"\"Test that /-/allowed.json returns the expected structure.\"\"\"\n    response = await ds_with_permissions.client.get(\n        \"/-/allowed.json?action=view-instance\"\n    )\n    assert response.status_code == 200\n    data = response.json()\n\n    # Check required fields\n    assert \"action\" in data\n    assert \"actor_id\" in data\n    assert \"page\" in data\n    assert \"page_size\" in data\n    assert \"total\" in data\n    assert \"items\" in data\n\n    # Check items structure\n    assert isinstance(data[\"items\"], list)\n    if data[\"items\"]:\n        item = data[\"items\"][0]\n        assert \"parent\" in item\n        assert \"child\" in item\n        assert \"resource\" in item\n\n\n@pytest.mark.asyncio\nasync def test_allowed_json_with_actor(ds_with_permissions):\n    \"\"\"Test /-/allowed.json includes actor information.\"\"\"\n    response = await ds_with_permissions.client.get(\n        \"/-/allowed.json?action=view-table\",\n        cookies={\n            \"ds_actor\": ds_with_permissions.client.actor_cookie({\"id\": \"test_user\"})\n        },\n    )\n    assert response.status_code == 200\n    data = response.json()\n    assert data[\"actor_id\"] == \"test_user\"\n\n\n@pytest.mark.asyncio\nasync def test_allowed_json_pagination():\n    \"\"\"Test that /-/allowed.json pagination works.\"\"\"\n    ds = Datasette()\n    await ds.invoke_startup()\n\n    # Create many tables to test pagination\n    db = ds.add_memory_database(\"test\")\n    for i in range(30):\n        await db.execute_write(f\"CREATE TABLE table{i:02d} (id INTEGER PRIMARY KEY)\")\n    await ds.refresh_schemas()\n\n    # Test page 1\n    response = await ds.client.get(\n        \"/-/allowed.json?action=view-table&page_size=10&page=1\"\n    )\n    assert response.status_code == 200\n    data = response.json()\n    assert data[\"page\"] == 1\n    assert data[\"page_size\"] == 10\n    assert len(data[\"items\"]) == 10\n\n    # Test page 2\n    response = await ds.client.get(\n        \"/-/allowed.json?action=view-table&page_size=10&page=2\"\n    )\n    assert response.status_code == 200\n    data = response.json()\n    assert data[\"page\"] == 2\n    assert len(data[\"items\"]) == 10\n\n    # Verify items are different between pages\n    response1 = await ds.client.get(\n        \"/-/allowed.json?action=view-table&page_size=10&page=1\"\n    )\n    response2 = await ds.client.get(\n        \"/-/allowed.json?action=view-table&page_size=10&page=2\"\n    )\n    items1 = {(item[\"parent\"], item[\"child\"]) for item in response1.json()[\"items\"]}\n    items2 = {(item[\"parent\"], item[\"child\"]) for item in response2.json()[\"items\"]}\n    assert items1 != items2\n\n\n@pytest.mark.asyncio\nasync def test_allowed_json_total_count(tmp_path_factory):\n    \"\"\"Test that /-/allowed.json returns correct total count.\"\"\"\n    from datasette.database import Database\n\n    # Use temporary file databases to avoid leakage from other tests\n    tmp_dir = tmp_path_factory.mktemp(\"test_allowed_json_total_count\")\n\n    ds = Datasette()\n    await ds.invoke_startup()\n\n    # Create test databases with tables\n    analytics_db = ds.add_database(\n        Database(ds, path=str(tmp_dir / \"analytics.db\")), name=\"analytics\"\n    )\n    await analytics_db.execute_write(\n        \"CREATE TABLE IF NOT EXISTS users (id INTEGER PRIMARY KEY, name TEXT, email TEXT)\"\n    )\n    await analytics_db.execute_write(\n        \"CREATE TABLE IF NOT EXISTS events (id INTEGER PRIMARY KEY, event_type TEXT, user_id INTEGER)\"\n    )\n\n    production_db = ds.add_database(\n        Database(ds, path=str(tmp_dir / \"production.db\")), name=\"production\"\n    )\n    await production_db.execute_write(\n        \"CREATE TABLE IF NOT EXISTS orders (id INTEGER PRIMARY KEY, total REAL)\"\n    )\n    await production_db.execute_write(\n        \"CREATE TABLE IF NOT EXISTS customers (id INTEGER PRIMARY KEY, name TEXT)\"\n    )\n\n    await ds.refresh_schemas()\n\n    response = await ds.client.get(\"/-/allowed.json?action=view-table\")\n    assert response.status_code == 200\n    data = response.json()\n\n    # We created 4 tables total (2 in analytics, 2 in production)\n    import json\n\n    assert (\n        data[\"total\"] == 4\n    ), f\"Expected total=4, got: {json.dumps(data, separators=(',', ':'))}\"\n\n\n# /-/rules.json tests\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"path,expected_status,expected_keys\",\n    [\n        # Instance level rules\n        (\n            \"/-/rules.json?action=view-instance\",\n            200,\n            {\"action\", \"items\", \"total\", \"page\"},\n        ),\n        # Database level rules\n        (\n            \"/-/rules.json?action=view-database\",\n            200,\n            {\"action\", \"items\", \"total\", \"page\"},\n        ),\n        # Table level rules\n        (\n            \"/-/rules.json?action=view-table\",\n            200,\n            {\"action\", \"items\", \"total\", \"page\"},\n        ),\n        # Missing action parameter\n        (\"/-/rules.json\", 400, {\"error\"}),\n        # Invalid action\n        (\"/-/rules.json?action=nonexistent\", 404, {\"error\"}),\n    ],\n)\nasync def test_rules_json_basic(\n    ds_with_permissions, path, expected_status, expected_keys\n):\n    # Use root actor for rules endpoint (requires permissions-debug)\n    response = await ds_with_permissions.client.get(\n        path,\n        cookies={\"ds_actor\": ds_with_permissions.client.actor_cookie({\"id\": \"root\"})},\n    )\n    assert response.status_code == expected_status\n    data = response.json()\n    assert expected_keys.issubset(data.keys())\n\n\n@pytest.mark.asyncio\nasync def test_rules_json_response_structure(ds_with_permissions):\n    \"\"\"Test that /-/rules.json returns the expected structure.\"\"\"\n    response = await ds_with_permissions.client.get(\n        \"/-/rules.json?action=view-instance\",\n        cookies={\"ds_actor\": ds_with_permissions.client.actor_cookie({\"id\": \"root\"})},\n    )\n    assert response.status_code == 200\n    data = response.json()\n\n    # Check required fields\n    assert \"action\" in data\n    assert \"actor_id\" in data\n    assert \"page\" in data\n    assert \"page_size\" in data\n    assert \"total\" in data\n    assert \"items\" in data\n\n    # Check items structure\n    assert isinstance(data[\"items\"], list)\n    if data[\"items\"]:\n        item = data[\"items\"][0]\n        assert \"parent\" in item\n        assert \"child\" in item\n        assert \"resource\" in item\n        assert \"allow\" in item\n        assert \"reason\" in item\n\n\n@pytest.mark.asyncio\nasync def test_rules_json_includes_all_rules(ds_with_permissions):\n    \"\"\"Test that /-/rules.json includes both allowed and denied resources.\"\"\"\n    # Root user should see rules for everything\n    response = await ds_with_permissions.client.get(\n        \"/-/rules.json?action=view-table\",\n        cookies={\"ds_actor\": ds_with_permissions.client.actor_cookie({\"id\": \"root\"})},\n    )\n    assert response.status_code == 200\n    data = response.json()\n\n    # Should have items (root has global allow)\n    assert len(data[\"items\"]) > 0\n\n    # Each item should have allow field (0 or 1)\n    for item in data[\"items\"]:\n        assert \"allow\" in item\n        assert item[\"allow\"] in [0, 1]\n\n\n@pytest.mark.asyncio\nasync def test_rules_json_pagination():\n    \"\"\"Test that /-/rules.json pagination works.\"\"\"\n    ds = Datasette()\n    ds.root_enabled = True\n    await ds.invoke_startup()\n\n    # Create some tables\n    db = ds.add_memory_database(\"test\")\n    for i in range(5):\n        await db.execute_write(\n            f\"CREATE TABLE IF NOT EXISTS table{i:02d} (id INTEGER PRIMARY KEY)\"\n        )\n    await ds.refresh_schemas()\n\n    # Test basic pagination structure - just verify it returns paginated results\n    response = await ds.client.get(\n        \"/-/rules.json?action=view-table&page_size=2&page=1\",\n        cookies={\"ds_actor\": ds.client.actor_cookie({\"id\": \"root\"})},\n    )\n    assert response.status_code == 200\n    data = response.json()\n    assert data[\"page\"] == 1\n    assert data[\"page_size\"] == 2\n    # Verify items is a list (may have fewer items than page_size if there aren't many rules)\n    assert isinstance(data[\"items\"], list)\n    assert \"total\" in data\n\n\n@pytest.mark.asyncio\nasync def test_rules_json_with_actor(ds_with_permissions):\n    \"\"\"Test /-/rules.json includes actor information.\"\"\"\n    # Use root actor (rules endpoint requires permissions-debug)\n    response = await ds_with_permissions.client.get(\n        \"/-/rules.json?action=view-table\",\n        cookies={\"ds_actor\": ds_with_permissions.client.actor_cookie({\"id\": \"root\"})},\n    )\n    assert response.status_code == 200\n    data = response.json()\n    assert data[\"actor_id\"] == \"root\"\n\n\n@pytest.mark.asyncio\nasync def test_root_user_respects_settings_deny():\n    \"\"\"\n    Test for issue #2509: Settings-based deny rules should override root user privileges.\n\n    When a database has `allow: false` in settings, the root user should NOT see\n    that database in /-/allowed.json?action=view-database.\n    \"\"\"\n    ds = Datasette(\n        config={\n            \"databases\": {\n                \"content\": {\n                    \"allow\": False,  # Deny everyone, including root\n                }\n            }\n        }\n    )\n    ds.root_enabled = True\n    await ds.invoke_startup()\n    ds.add_memory_database(\"content\")\n\n    # Root user should NOT see the denied database\n    response = await ds.client.get(\n        \"/-/allowed.json?action=view-database\",\n        cookies={\"ds_actor\": ds.client.actor_cookie({\"id\": \"root\"})},\n    )\n    assert response.status_code == 200\n    data = response.json()\n\n    # Check that content database is NOT in the allowed list\n    allowed_databases = [item[\"parent\"] for item in data[\"items\"]]\n    assert \"content\" not in allowed_databases, (\n        f\"Root user should not see 'content' database when settings deny it, \"\n        f\"but found it in: {allowed_databases}\"\n    )\n\n\n@pytest.mark.asyncio\nasync def test_root_user_respects_settings_deny_tables():\n    \"\"\"\n    Test for issue #2509: Settings-based deny rules should override root for tables too.\n\n    When a database has `allow: false` in settings, the root user should NOT see\n    tables from that database in /-/allowed.json?action=view-table.\n    \"\"\"\n    ds = Datasette(\n        config={\n            \"databases\": {\n                \"content\": {\n                    \"allow\": False,  # Deny everyone, including root\n                }\n            }\n        }\n    )\n    ds.root_enabled = True\n    await ds.invoke_startup()\n\n    # Add a database with a table\n    db = ds.add_memory_database(\"content\")\n    await db.execute_write(\"CREATE TABLE repos (id INTEGER PRIMARY KEY, name TEXT)\")\n    await ds.refresh_schemas()\n\n    # Root user should NOT see tables from the content database\n    response = await ds.client.get(\n        \"/-/allowed.json?action=view-table\",\n        cookies={\"ds_actor\": ds.client.actor_cookie({\"id\": \"root\"})},\n    )\n    assert response.status_code == 200\n    data = response.json()\n\n    # Check that content.repos table is NOT in the allowed list\n    content_tables = [\n        item[\"child\"] for item in data[\"items\"] if item[\"parent\"] == \"content\"\n    ]\n    assert \"repos\" not in content_tables, (\n        f\"Root user should not see tables from 'content' database when settings deny it, \"\n        f\"but found: {content_tables}\"\n    )\n\n\n@pytest.mark.asyncio\nasync def test_execute_sql_requires_view_database():\n    \"\"\"\n    Test for issue #2527: execute-sql permission should require view-database permission.\n\n    A user who has execute-sql permission but not view-database permission should not\n    be able to execute SQL on that database.\n    \"\"\"\n    from datasette.permissions import PermissionSQL\n    from datasette import hookimpl\n\n    class TestPermissionPlugin:\n        __name__ = \"TestPermissionPlugin\"\n\n        @hookimpl\n        def permission_resources_sql(self, datasette, actor, action):\n            if actor is None or actor.get(\"id\") != \"test_user\":\n                return []\n\n            if action == \"execute-sql\":\n                # Grant execute-sql on the \"secret\" database\n                return PermissionSQL(\n                    sql=\"SELECT 'secret' AS parent, NULL AS child, 1 AS allow, 'can execute sql' AS reason\",\n                )\n            elif action == \"view-database\":\n                # Deny view-database on the \"secret\" database\n                return PermissionSQL(\n                    sql=\"SELECT 'secret' AS parent, NULL AS child, 0 AS allow, 'cannot view db' AS reason\",\n                )\n\n            return []\n\n    plugin = TestPermissionPlugin()\n\n    ds = Datasette()\n    await ds.invoke_startup()\n    ds.pm.register(plugin, name=\"test_plugin\")\n\n    try:\n        ds.add_memory_database(\"secret\")\n        await ds.refresh_schemas()\n\n        # User should NOT have execute-sql permission because view-database is denied\n        response = await ds.client.get(\n            \"/-/allowed.json?action=execute-sql\",\n            cookies={\"ds_actor\": ds.client.actor_cookie({\"id\": \"test_user\"})},\n        )\n        assert response.status_code == 200\n        data = response.json()\n\n        # The \"secret\" database should NOT be in the allowed list for execute-sql\n        allowed_databases = [item[\"parent\"] for item in data[\"items\"]]\n        assert \"secret\" not in allowed_databases, (\n            f\"User should not have execute-sql permission without view-database, \"\n            f\"but found 'secret' in: {allowed_databases}\"\n        )\n\n        # Also verify that attempting to execute SQL on the database is denied\n        # (may be 403 or 302 redirect to login/error page depending on middleware)\n        response = await ds.client.get(\n            \"/secret?sql=SELECT+1\",\n            cookies={\"ds_actor\": ds.client.actor_cookie({\"id\": \"test_user\"})},\n        )\n        assert response.status_code in (302, 403), (\n            f\"Expected 302 or 403 when trying to execute SQL without view-database permission, \"\n            f\"but got {response.status_code}\"\n        )\n    finally:\n        ds.pm.unregister(plugin)\n"
  },
  {
    "path": "tests/test_permissions.py",
    "content": "import collections\nfrom datasette.app import Datasette\nfrom datasette.cli import cli\nfrom datasette.default_permissions import restrictions_allow_action\nfrom .fixtures import assert_permissions_checked, make_app_client\nfrom click.testing import CliRunner\nfrom bs4 import BeautifulSoup as Soup\nimport copy\nimport json\nfrom pprint import pprint\nimport pytest_asyncio\nimport pytest\nimport re\nimport time\nimport urllib\n\n\n@pytest.fixture(scope=\"module\")\ndef padlock_client():\n    with make_app_client(\n        config={\n            \"databases\": {\n                \"fixtures\": {\n                    \"queries\": {\"two\": {\"sql\": \"select 1 + 1\"}},\n                }\n            }\n        }\n    ) as client:\n        yield client\n\n\n@pytest_asyncio.fixture\nasync def perms_ds():\n    ds = Datasette()\n    await ds.invoke_startup()\n    one = ds.add_memory_database(\"perms_ds_one\")\n    two = ds.add_memory_database(\"perms_ds_two\")\n    await one.execute_write(\"create table if not exists t1 (id integer primary key)\")\n    await one.execute_write(\"insert or ignore into t1 (id) values (1)\")\n    await one.execute_write(\"create view if not exists v1 as select * from t1\")\n    await one.execute_write(\"create table if not exists t2 (id integer primary key)\")\n    await two.execute_write(\"create table if not exists t1 (id integer primary key)\")\n    # Trigger catalog refresh so allowed_resources() can be called\n    await ds.client.get(\"/\")\n    return ds\n\n\n@pytest.mark.parametrize(\n    \"allow,expected_anon,expected_auth\",\n    [\n        (None, 200, 200),\n        ({}, 403, 403),\n        ({\"id\": \"root\"}, 403, 200),\n    ],\n)\n@pytest.mark.parametrize(\n    \"path\",\n    (\n        \"/\",\n        \"/fixtures\",\n        \"/-/api\",\n        \"/fixtures/compound_three_primary_keys\",\n        \"/fixtures/compound_three_primary_keys/a,a,a\",\n        pytest.param(\n            \"/fixtures/two\",\n            marks=pytest.mark.xfail(\n                reason=\"view-query not yet migrated to new permission system\"\n            ),\n        ),  # Query\n    ),\n)\ndef test_view_padlock(allow, expected_anon, expected_auth, path, padlock_client):\n    padlock_client.ds.config[\"allow\"] = allow\n    fragment = \"🔒\"\n    anon_response = padlock_client.get(path)\n    assert expected_anon == anon_response.status\n    if allow and anon_response.status == 200:\n        # Should be no padlock\n        assert fragment not in anon_response.text\n    auth_response = padlock_client.get(\n        path,\n        cookies={\"ds_actor\": padlock_client.actor_cookie({\"id\": \"root\"})},\n    )\n    assert expected_auth == auth_response.status\n    # Check for the padlock\n    if allow and expected_anon == 403 and expected_auth == 200:\n        assert fragment in auth_response.text\n    del padlock_client.ds.config[\"allow\"]\n\n\n@pytest.mark.parametrize(\n    \"allow,expected_anon,expected_auth\",\n    [\n        (None, 200, 200),\n        ({}, 403, 403),\n        ({\"id\": \"root\"}, 403, 200),\n    ],\n)\n@pytest.mark.parametrize(\"use_metadata\", (True, False))\ndef test_view_database(allow, expected_anon, expected_auth, use_metadata):\n    key = \"metadata\" if use_metadata else \"config\"\n    kwargs = {key: {\"databases\": {\"fixtures\": {\"allow\": allow}}}}\n    with make_app_client(**kwargs) as client:\n        for path in (\n            \"/fixtures\",\n            \"/fixtures/compound_three_primary_keys\",\n            \"/fixtures/compound_three_primary_keys/a,a,a\",\n        ):\n            anon_response = client.get(path)\n            assert expected_anon == anon_response.status, path\n            if allow and path == \"/fixtures\" and anon_response.status == 200:\n                # Should be no padlock\n                assert \">fixtures 🔒\" not in anon_response.text\n            auth_response = client.get(\n                path,\n                cookies={\"ds_actor\": client.actor_cookie({\"id\": \"root\"})},\n            )\n            assert expected_auth == auth_response.status\n            if (\n                allow\n                and path == \"/fixtures\"\n                and expected_anon == 403\n                and expected_auth == 200\n            ):\n                assert \">fixtures 🔒\" in auth_response.text\n\n\ndef test_database_list_respects_view_database():\n    with make_app_client(\n        config={\"databases\": {\"fixtures\": {\"allow\": {\"id\": \"root\"}}}},\n        extra_databases={\"data.db\": \"create table names (name text)\"},\n    ) as client:\n        anon_response = client.get(\"/\")\n        assert 'data' in anon_response.text\n        assert 'fixtures' not in anon_response.text\n        auth_response = client.get(\n            \"/\",\n            cookies={\"ds_actor\": client.actor_cookie({\"id\": \"root\"})},\n        )\n        assert 'data' in auth_response.text\n        assert 'fixtures 🔒' in auth_response.text\n\n\ndef test_database_list_respects_view_table():\n    with make_app_client(\n        config={\n            \"databases\": {\n                \"data\": {\n                    \"tables\": {\n                        \"names\": {\"allow\": {\"id\": \"root\"}},\n                        \"v\": {\"allow\": {\"id\": \"root\"}},\n                    }\n                }\n            }\n        },\n        extra_databases={\n            \"data.db\": \"create table names (name text); create view v as select * from names\"\n        },\n    ) as client:\n        html_fragments = [\n            \">names 🔒\",\n            \">v 🔒\",\n        ]\n        anon_response_text = client.get(\"/\").text\n        assert \"0 rows in 0 tables\" in anon_response_text\n        for html_fragment in html_fragments:\n            assert html_fragment not in anon_response_text\n        auth_response_text = client.get(\n            \"/\",\n            cookies={\"ds_actor\": client.actor_cookie({\"id\": \"root\"})},\n        ).text\n        for html_fragment in html_fragments:\n            assert html_fragment in auth_response_text\n\n\n@pytest.mark.parametrize(\n    \"allow,expected_anon,expected_auth\",\n    [\n        (None, 200, 200),\n        ({}, 403, 403),\n        ({\"id\": \"root\"}, 403, 200),\n    ],\n)\n@pytest.mark.parametrize(\"use_metadata\", (True, False))\ndef test_view_table(allow, expected_anon, expected_auth, use_metadata):\n    key = \"metadata\" if use_metadata else \"config\"\n    kwargs = {\n        key: {\n            \"databases\": {\n                \"fixtures\": {\n                    \"tables\": {\"compound_three_primary_keys\": {\"allow\": allow}}\n                }\n            }\n        }\n    }\n    with make_app_client(**kwargs) as client:\n        anon_response = client.get(\"/fixtures/compound_three_primary_keys\")\n        assert expected_anon == anon_response.status\n        if allow and anon_response.status == 200:\n            # Should be no padlock\n            assert \">compound_three_primary_keys 🔒\" not in anon_response.text\n        auth_response = client.get(\n            \"/fixtures/compound_three_primary_keys\",\n            cookies={\"ds_actor\": client.actor_cookie({\"id\": \"root\"})},\n        )\n        assert expected_auth == auth_response.status\n        if allow and expected_anon == 403 and expected_auth == 200:\n            assert \">compound_three_primary_keys 🔒\" in auth_response.text\n\n\ndef test_table_list_respects_view_table():\n    with make_app_client(\n        config={\n            \"databases\": {\n                \"fixtures\": {\n                    \"tables\": {\n                        \"compound_three_primary_keys\": {\"allow\": {\"id\": \"root\"}},\n                        # And a SQL view too:\n                        \"paginated_view\": {\"allow\": {\"id\": \"root\"}},\n                    }\n                }\n            }\n        }\n    ) as client:\n        html_fragments = [\n            \">compound_three_primary_keys 🔒\",\n            \">paginated_view 🔒\",\n        ]\n        anon_response = client.get(\"/fixtures\")\n        for html_fragment in html_fragments:\n            assert html_fragment not in anon_response.text\n        auth_response = client.get(\n            \"/fixtures\", cookies={\"ds_actor\": client.actor_cookie({\"id\": \"root\"})}\n        )\n        for html_fragment in html_fragments:\n            assert html_fragment in auth_response.text\n\n\n@pytest.mark.parametrize(\n    \"allow,expected_anon,expected_auth\",\n    [\n        (None, 200, 200),\n        ({}, 403, 403),\n        ({\"id\": \"root\"}, 403, 200),\n    ],\n)\ndef test_view_query(allow, expected_anon, expected_auth):\n    with make_app_client(\n        config={\n            \"databases\": {\n                \"fixtures\": {\"queries\": {\"q\": {\"sql\": \"select 1 + 1\", \"allow\": allow}}}\n            }\n        }\n    ) as client:\n        anon_response = client.get(\"/fixtures/q\")\n        assert expected_anon == anon_response.status\n        if allow and anon_response.status == 200:\n            # Should be no padlock\n            assert \"🔒\" not in anon_response.text\n        auth_response = client.get(\n            \"/fixtures/q\", cookies={\"ds_actor\": client.actor_cookie({\"id\": \"root\"})}\n        )\n        assert expected_auth == auth_response.status\n        if allow and expected_anon == 403 and expected_auth == 200:\n            assert \">fixtures: q 🔒\" in auth_response.text\n\n\n@pytest.mark.parametrize(\n    \"config\",\n    [\n        {\"allow_sql\": {\"id\": \"root\"}},\n        {\"databases\": {\"fixtures\": {\"allow_sql\": {\"id\": \"root\"}}}},\n    ],\n)\ndef test_execute_sql(config):\n    schema_re = re.compile(\"const schema = ({.*?});\", re.DOTALL)\n    with make_app_client(config=config) as client:\n        form_fragment = '',\n        '',\n        '',\n    ):\n        assert fragment in response.text\n    # Should show one failure and one success\n    soup = Soup(response.text, \"html.parser\")\n    table = soup.find(\"table\", {\"id\": \"permission-checks-table\"})\n    rows = table.find(\"tbody\").find_all(\"tr\")\n    checks = []\n    for row in rows:\n        cells = row.find_all(\"td\")\n        result_cell = cells[5]\n        if result_cell.select_one(\".check-result-true\"):\n            result = True\n        elif result_cell.select_one(\".check-result-false\"):\n            result = False\n        else:\n            result = None\n        actor_code = cells[4].find(\"code\")\n        actor = json.loads(actor_code.text) if actor_code else None\n        checks.append(\n            {\n                \"action\": cells[1].text.strip(),\n                \"result\": result,\n                \"actor\": actor,\n            }\n        )\n    expected_checks = [\n        {\n            \"action\": \"permissions-debug\",\n            \"result\": True,\n            \"actor\": {\"id\": \"root\"},\n        },\n        {\n            \"action\": \"view-instance\",\n            \"result\": True,\n            \"actor\": {\"id\": \"root\"},\n        },\n        {\"action\": \"debug-menu\", \"result\": False, \"actor\": None},\n        {\n            \"action\": \"view-instance\",\n            \"result\": True,\n            \"actor\": None,\n        },\n        {\n            \"action\": \"permissions-debug\",\n            \"result\": False,\n            \"actor\": None,\n        },\n        {\n            \"action\": \"view-instance\",\n            \"result\": True,\n            \"actor\": None,\n        },\n    ]\n    if filter_ == \"only-yours\":\n        expected_checks = [\n            check for check in expected_checks if check[\"actor\"] is not None\n        ]\n    elif filter_ == \"exclude-yours\":\n        expected_checks = [check for check in expected_checks if check[\"actor\"] is None]\n    assert checks == expected_checks\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"actor,allow,expected_fragment\",\n    [\n        ('{\"id\":\"root\"}', \"{}\", \"Result: deny\"),\n        ('{\"id\":\"root\"}', '{\"id\": \"*\"}', \"Result: allow\"),\n        ('{\"', '{\"id\": \"*\"}', \"Actor JSON error\"),\n        ('{\"id\":\"root\"}', '\"*\"}', \"Allow JSON error\"),\n    ],\n)\nasync def test_allow_debug(ds_client, actor, allow, expected_fragment):\n    response = await ds_client.get(\n        \"/-/allow-debug?\" + urllib.parse.urlencode({\"actor\": actor, \"allow\": allow})\n    )\n    assert response.status_code == 200\n    assert expected_fragment in response.text\n\n\n@pytest.mark.parametrize(\n    \"allow,expected\",\n    [\n        ({\"id\": \"root\"}, 403),\n        ({\"id\": \"root\", \"unauthenticated\": True}, 200),\n    ],\n)\ndef test_allow_unauthenticated(allow, expected):\n    with make_app_client(config={\"allow\": allow}) as client:\n        assert expected == client.get(\"/\").status\n\n\n@pytest.fixture(scope=\"session\")\ndef view_instance_client():\n    with make_app_client(config={\"allow\": {}}) as client:\n        yield client\n\n\n@pytest.mark.parametrize(\n    \"path\",\n    [\n        \"/\",\n        \"/fixtures\",\n        \"/fixtures/facetable\",\n        \"/-/versions\",\n        \"/-/plugins\",\n        \"/-/settings\",\n        \"/-/threads\",\n        \"/-/databases\",\n        \"/-/permissions\",\n        \"/-/messages\",\n        \"/-/patterns\",\n    ],\n)\ndef test_view_instance(path, view_instance_client):\n    assert 403 == view_instance_client.get(path).status\n    if path not in (\"/-/permissions\", \"/-/messages\", \"/-/patterns\"):\n        assert 403 == view_instance_client.get(path + \".json\").status\n\n\n@pytest.fixture(scope=\"session\")\ndef cascade_app_client():\n    with make_app_client(is_immutable=True) as client:\n        yield client\n\n\n@pytest.mark.parametrize(\n    \"path,permissions,expected_status\",\n    [\n        (\"/\", [], 403),\n        (\"/\", [\"instance\"], 200),\n        # Can view table even if not allowed database or instance\n        (\"/fixtures/binary_data\", [], 403),\n        (\"/fixtures/binary_data\", [\"database\"], 403),\n        (\"/fixtures/binary_data\", [\"instance\"], 403),\n        (\"/fixtures/binary_data\", [\"table\"], 200),\n        (\"/fixtures/binary_data\", [\"table\", \"database\"], 200),\n        (\"/fixtures/binary_data\", [\"table\", \"database\", \"instance\"], 200),\n        # ... same for row\n        (\"/fixtures/binary_data/1\", [], 403),\n        (\"/fixtures/binary_data/1\", [\"database\"], 403),\n        (\"/fixtures/binary_data/1\", [\"instance\"], 403),\n        (\"/fixtures/binary_data/1\", [\"table\"], 200),\n        (\"/fixtures/binary_data/1\", [\"table\", \"database\"], 200),\n        (\"/fixtures/binary_data/1\", [\"table\", \"database\", \"instance\"], 200),\n        # Can view query even if not allowed database or instance\n        (\"/fixtures/magic_parameters\", [], 403),\n        (\"/fixtures/magic_parameters\", [\"database\"], 403),\n        (\"/fixtures/magic_parameters\", [\"instance\"], 403),\n        (\"/fixtures/magic_parameters\", [\"query\"], 200),\n        (\"/fixtures/magic_parameters\", [\"query\", \"database\"], 200),\n        (\"/fixtures/magic_parameters\", [\"query\", \"database\", \"instance\"], 200),\n        # Can view database even if not allowed instance\n        (\"/fixtures\", [], 403),\n        (\"/fixtures\", [\"instance\"], 403),\n        (\"/fixtures\", [\"database\"], 200),\n        # Downloading the fixtures.db file\n        (\"/fixtures.db\", [], 403),\n        (\"/fixtures.db\", [\"instance\"], 403),\n        (\"/fixtures.db\", [\"database\"], 200),\n        (\"/fixtures.db\", [\"download\"], 200),\n    ],\n)\ndef test_permissions_cascade(cascade_app_client, path, permissions, expected_status):\n    \"\"\"Test that e.g. having view-table but NOT view-database lets you view table page, etc\"\"\"\n    allow = {\"id\": \"*\"}\n    deny = {}\n    previous_config = cascade_app_client.ds.config\n    updated_config = copy.deepcopy(previous_config)\n    actor = {\"id\": \"test\"}\n    if \"download\" in permissions:\n        actor[\"can_download\"] = 1\n    try:\n        # Set up the different allow blocks\n        updated_config[\"allow\"] = allow if \"instance\" in permissions else deny\n        # Note: download permission also needs database access (via plugin granting both)\n        # so we don't set a deny rule when download is in permissions\n        updated_config[\"databases\"][\"fixtures\"][\"allow\"] = (\n            allow if (\"database\" in permissions or \"download\" in permissions) else deny\n        )\n        updated_config[\"databases\"][\"fixtures\"][\"tables\"][\"binary_data\"] = {\n            \"allow\": (allow if \"table\" in permissions else deny)\n        }\n        updated_config[\"databases\"][\"fixtures\"][\"queries\"][\"magic_parameters\"][\n            \"allow\"\n        ] = (allow if \"query\" in permissions else deny)\n        cascade_app_client.ds.config = updated_config\n        response = cascade_app_client.get(\n            path,\n            cookies={\"ds_actor\": cascade_app_client.actor_cookie(actor)},\n        )\n        assert (\n            response.status == expected_status\n        ), \"path: {}, permissions: {}, expected_status: {}, status: {}\".format(\n            path, permissions, expected_status, response.status\n        )\n    finally:\n        cascade_app_client.ds.config = previous_config\n\n\ndef test_padlocks_on_database_page(cascade_app_client):\n    config = {\n        \"databases\": {\n            \"fixtures\": {\n                \"allow\": {\"id\": \"test\"},\n                \"tables\": {\n                    \"123_starts_with_digits\": {\"allow\": True},\n                    \"simple_view\": {\"allow\": True},\n                },\n                \"queries\": {\"query_two\": {\"allow\": True, \"sql\": \"select 2\"}},\n            }\n        }\n    }\n    previous_config = cascade_app_client.ds.config\n    try:\n        cascade_app_client.ds.config = config\n        response = cascade_app_client.get(\n            \"/fixtures\",\n            cookies={\"ds_actor\": cascade_app_client.actor_cookie({\"id\": \"test\"})},\n        )\n        # Tables\n        assert \">123_starts_with_digits\" in response.text\n        assert \">Table With Space In Name 🔒\" in response.text\n        # Queries\n        assert \">from_async_hook 🔒\" in response.text\n        assert \">query_two\" in response.text\n        # Views\n        assert \">paginated_view 🔒\" in response.text\n        assert \">simple_view\" in response.text\n    finally:\n        cascade_app_client.ds.config = previous_config\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"actor,permission,resource_1,resource_2,expected_result\",\n    (\n        # Without restrictions the defaults apply\n        ({\"id\": \"t\"}, \"view-instance\", None, None, True),\n        ({\"id\": \"t\"}, \"view-database\", \"one\", None, True),\n        ({\"id\": \"t\"}, \"view-table\", \"one\", \"t1\", True),\n        # If there is an _r block, everything gets denied unless explicitly allowed\n        ({\"id\": \"t\", \"_r\": {}}, \"view-instance\", None, None, False),\n        ({\"id\": \"t\", \"_r\": {}}, \"view-database\", \"one\", None, False),\n        ({\"id\": \"t\", \"_r\": {}}, \"view-table\", \"one\", \"t1\", False),\n        # Explicit allowing works at the \"a\" for all level:\n        ({\"id\": \"t\", \"_r\": {\"a\": [\"vi\"]}}, \"view-instance\", None, None, True),\n        ({\"id\": \"t\", \"_r\": {\"a\": [\"vd\"]}}, \"view-database\", \"one\", None, True),\n        ({\"id\": \"t\", \"_r\": {\"a\": [\"vt\"]}}, \"view-table\", \"one\", \"t1\", True),\n        # But not if it's the wrong permission\n        ({\"id\": \"t\", \"_r\": {\"a\": [\"vi\"]}}, \"view-database\", \"one\", None, False),\n        ({\"id\": \"t\", \"_r\": {\"a\": [\"vd\"]}}, \"view-table\", \"one\", \"t1\", False),\n        # Works at the \"d\" for database level:\n        ({\"id\": \"t\", \"_r\": {\"d\": {\"one\": [\"vd\"]}}}, \"view-database\", \"one\", None, True),\n        (\n            # view-database-download requires view-database too (also_requires)\n            {\"id\": \"t\", \"_r\": {\"d\": {\"one\": [\"vdd\", \"vd\"]}}},\n            \"view-database-download\",\n            \"one\",\n            None,\n            True,\n        ),\n        (\n            # execute-sql requires view-database too (also_requires)\n            {\"id\": \"t\", \"_r\": {\"d\": {\"one\": [\"es\", \"vd\"]}}},\n            \"execute-sql\",\n            \"one\",\n            None,\n            True,\n        ),\n        # Works at the \"r\" for table level:\n        (\n            {\"id\": \"t\", \"_r\": {\"r\": {\"one\": {\"t1\": [\"vt\"]}}}},\n            \"view-table\",\n            \"one\",\n            \"t1\",\n            True,\n        ),\n        (\n            {\"id\": \"t\", \"_r\": {\"r\": {\"one\": {\"t1\": [\"vt\"]}}}},\n            \"view-table\",\n            \"one\",\n            \"t2\",\n            False,\n        ),\n        # non-abbreviations should work too\n        (\n            {\"id\": \"t\", \"_r\": {\"a\": [\"view-instance\"]}},\n            \"view-instance\",\n            None,\n            None,\n            True,\n        ),\n        (\n            {\"id\": \"t\", \"_r\": {\"d\": {\"one\": [\"view-database\"]}}},\n            \"view-database\",\n            \"one\",\n            None,\n            True,\n        ),\n        (\n            {\"id\": \"t\", \"_r\": {\"r\": {\"one\": {\"t1\": [\"view-table\"]}}}},\n            \"view-table\",\n            \"one\",\n            \"t1\",\n            True,\n        ),\n        # view-database does NOT grant view-instance (no upward cascading)\n        ({\"id\": \"t\", \"_r\": {\"a\": [\"vd\"]}}, \"view-instance\", None, None, False),\n    ),\n)\nasync def test_actor_restricted_permissions(\n    perms_ds, actor, permission, resource_1, resource_2, expected_result\n):\n    perms_ds.pdb = True\n    perms_ds.root_enabled = True  # Allow root actor to access /-/permissions\n    cookies = {\"ds_actor\": perms_ds.sign({\"a\": {\"id\": \"root\"}}, \"actor\")}\n    csrftoken = (await perms_ds.client.get(\"/-/permissions\", cookies=cookies)).cookies[\n        \"ds_csrftoken\"\n    ]\n    cookies[\"ds_csrftoken\"] = csrftoken\n    response = await perms_ds.client.post(\n        \"/-/permissions\",\n        data={\n            \"actor\": json.dumps(actor),\n            \"permission\": permission,\n            \"resource_1\": resource_1,\n            \"resource_2\": resource_2,\n            \"csrftoken\": csrftoken,\n        },\n        cookies=cookies,\n    )\n    # Response mirrors /-/check JSON structure\n    if resource_1 is None:\n        expected_path = \"/\"\n    elif resource_2 is None:\n        expected_path = f\"/{resource_1}\"\n    else:\n        expected_path = f\"/{resource_1}/{resource_2}\"\n\n    expected_resource = {\n        \"parent\": resource_1,\n        \"child\": resource_2,\n        \"path\": expected_path,\n    }\n    expected = {\n        \"action\": permission,\n        \"allowed\": expected_result,\n        \"resource\": expected_resource,\n    }\n    if actor.get(\"id\"):\n        expected[\"actor_id\"] = actor[\"id\"]\n    assert response.json() == expected\n\n\nPermConfigTestCase = collections.namedtuple(\n    \"PermConfigTestCase\",\n    \"config,actor,action,resource,expected_result\",\n)\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"config,actor,action,resource,expected_result\",\n    (\n        # Simple view-instance default=True example\n        PermConfigTestCase(\n            config={},\n            actor=None,\n            action=\"view-instance\",\n            resource=None,\n            expected_result=True,\n        ),\n        # debug-menu on root\n        PermConfigTestCase(\n            config={\"permissions\": {\"debug-menu\": {\"id\": \"user\"}}},\n            actor={\"id\": \"user\"},\n            action=\"debug-menu\",\n            resource=None,\n            expected_result=True,\n        ),\n        # debug-menu on root, wrong actor\n        PermConfigTestCase(\n            config={\"permissions\": {\"debug-menu\": {\"id\": \"user\"}}},\n            actor={\"id\": \"user2\"},\n            action=\"debug-menu\",\n            resource=None,\n            expected_result=False,\n        ),\n        # create-table on root\n        PermConfigTestCase(\n            config={\"permissions\": {\"create-table\": {\"id\": \"user\"}}},\n            actor={\"id\": \"user\"},\n            action=\"create-table\",\n            resource=None,\n            expected_result=True,\n        ),\n        # create-table on database - no resource specified\n        PermConfigTestCase(\n            config={\n                \"databases\": {\n                    \"perms_ds_one\": {\"permissions\": {\"create-table\": {\"id\": \"user\"}}}\n                }\n            },\n            actor={\"id\": \"user\"},\n            action=\"create-table\",\n            resource=None,\n            expected_result=False,\n        ),\n        # create-table on database\n        PermConfigTestCase(\n            config={\n                \"databases\": {\n                    \"perms_ds_one\": {\"permissions\": {\"create-table\": {\"id\": \"user\"}}}\n                }\n            },\n            actor={\"id\": \"user\"},\n            action=\"create-table\",\n            resource=\"perms_ds_one\",\n            expected_result=True,\n        ),\n        # insert-row on root, wrong actor\n        PermConfigTestCase(\n            config={\"permissions\": {\"insert-row\": {\"id\": \"user\"}}},\n            actor={\"id\": \"user2\"},\n            action=\"insert-row\",\n            resource=(\"perms_ds_one\", \"t1\"),\n            expected_result=False,\n        ),\n        # insert-row on root, right actor\n        PermConfigTestCase(\n            config={\"permissions\": {\"insert-row\": {\"id\": \"user\"}}},\n            actor={\"id\": \"user\"},\n            action=\"insert-row\",\n            resource=(\"perms_ds_one\", \"t1\"),\n            expected_result=True,\n        ),\n        # set-column-type on specific table\n        PermConfigTestCase(\n            config={\n                \"databases\": {\n                    \"perms_ds_one\": {\n                        \"tables\": {\n                            \"t1\": {\"permissions\": {\"set-column-type\": {\"id\": \"user\"}}}\n                        }\n                    }\n                }\n            },\n            actor={\"id\": \"user\"},\n            action=\"set-column-type\",\n            resource=(\"perms_ds_one\", \"t1\"),\n            expected_result=True,\n        ),\n        # insert-row on database\n        PermConfigTestCase(\n            config={\n                \"databases\": {\n                    \"perms_ds_one\": {\"permissions\": {\"insert-row\": {\"id\": \"user\"}}}\n                }\n            },\n            actor={\"id\": \"user\"},\n            action=\"insert-row\",\n            resource=\"perms_ds_one\",\n            expected_result=True,\n        ),\n        # insert-row on table, wrong table\n        PermConfigTestCase(\n            config={\n                \"databases\": {\n                    \"perms_ds_one\": {\n                        \"tables\": {\n                            \"t1\": {\"permissions\": {\"insert-row\": {\"id\": \"user\"}}}\n                        }\n                    }\n                }\n            },\n            actor={\"id\": \"user\"},\n            action=\"insert-row\",\n            resource=(\"perms_ds_one\", \"t2\"),\n            expected_result=False,\n        ),\n        # insert-row on table, right table\n        PermConfigTestCase(\n            config={\n                \"databases\": {\n                    \"perms_ds_one\": {\n                        \"tables\": {\n                            \"t1\": {\"permissions\": {\"insert-row\": {\"id\": \"user\"}}}\n                        }\n                    }\n                }\n            },\n            actor={\"id\": \"user\"},\n            action=\"insert-row\",\n            resource=(\"perms_ds_one\", \"t1\"),\n            expected_result=True,\n        ),\n        # view-query on canned query, wrong actor\n        PermConfigTestCase(\n            config={\n                \"databases\": {\n                    \"perms_ds_one\": {\n                        \"queries\": {\n                            \"q1\": {\n                                \"sql\": \"select 1 + 1\",\n                                \"permissions\": {\"view-query\": {\"id\": \"user\"}},\n                            }\n                        }\n                    }\n                }\n            },\n            actor={\"id\": \"user2\"},\n            action=\"view-query\",\n            resource=(\"perms_ds_one\", \"q1\"),\n            expected_result=False,\n        ),\n        # view-query on canned query, right actor\n        PermConfigTestCase(\n            config={\n                \"databases\": {\n                    \"perms_ds_one\": {\n                        \"queries\": {\n                            \"q1\": {\n                                \"sql\": \"select 1 + 1\",\n                                \"permissions\": {\"view-query\": {\"id\": \"user\"}},\n                            }\n                        }\n                    }\n                }\n            },\n            actor={\"id\": \"user\"},\n            action=\"view-query\",\n            resource=(\"perms_ds_one\", \"q1\"),\n            expected_result=True,\n        ),\n    ),\n)\nasync def test_permissions_in_config(\n    perms_ds, config, actor, action, resource, expected_result\n):\n    previous_config = perms_ds.config\n    updated_config = copy.deepcopy(previous_config)\n    updated_config.update(config)\n    perms_ds.config = updated_config\n    try:\n        # Convert old-style resource to Resource object\n        from datasette.resources import DatabaseResource, TableResource\n\n        resource_obj = None\n        if resource:\n            if isinstance(resource, str):\n                resource_obj = DatabaseResource(database=resource)\n            elif isinstance(resource, tuple) and len(resource) == 2:\n                resource_obj = TableResource(database=resource[0], table=resource[1])\n\n        result = await perms_ds.allowed(\n            action=action, resource=resource_obj, actor=actor\n        )\n        if result != expected_result:\n            pprint(perms_ds._permission_checks)\n            assert result == expected_result\n    finally:\n        perms_ds.config = previous_config\n\n\n@pytest.mark.asyncio\nasync def test_allowed_resources_view_query_includes_actor_specific_canned_queries():\n    \"\"\"\n    Actor-specific canned queries should be listed by allowed_resources(\"view-query\").\n\n    This test is intentionally explicit about the previous bug:\n    - the canned query only exists for actor \"alice\"\n    - the permission rule only allows actor \"alice\" to view it\n    - allowed() succeeds for that specific query resource\n    - allowed_resources(\"view-query\", actor) must include the same query\n\n    Before the fix, QueryResource.resources_sql() called canned_queries(..., actor=None),\n    so the query was omitted from resource enumeration and allowed_resources() returned\n    an empty list even though allowed() returned True.\n    \"\"\"\n    from datasette import hookimpl\n    from datasette.permissions import PermissionSQL\n    from datasette.resources import QueryResource\n\n    class ActorSpecificQueryPlugin:\n        __name__ = \"ActorSpecificQueryPlugin\"\n\n        @hookimpl\n        def canned_queries(self, datasette, database, actor):\n            if database == \"testdb\" and actor and actor.get(\"id\") == \"alice\":\n                return {\"user_only\": {\"sql\": \"select 1 as n\"}}\n            return {}\n\n        @hookimpl\n        def permission_resources_sql(self, datasette, actor, action):\n            if action == \"view-query\" and actor and actor.get(\"id\") == \"alice\":\n                return PermissionSQL(sql=\"\"\"\n                        SELECT 'testdb' AS parent, 'user_only' AS child, 1 AS allow,\n                               'alice can view her actor-specific canned query' AS reason\n                    \"\"\")\n            return None\n\n    ds = Datasette(default_deny=True)\n    await ds.invoke_startup()\n    ds.add_memory_database(\"testdb\")\n    await ds._refresh_schemas()\n\n    plugin = ActorSpecificQueryPlugin()\n    ds.pm.register(plugin, name=\"actor_specific_query_plugin\")\n\n    try:\n        actor = {\"id\": \"alice\"}\n\n        assert await ds.allowed(\n            action=\"view-query\",\n            resource=QueryResource(\"testdb\", \"user_only\"),\n            actor=actor,\n        )\n\n        page = await ds.allowed_resources(\"view-query\", actor)\n        assert [(resource.parent, resource.child) for resource in page.resources] == [\n            (\"testdb\", \"user_only\")\n        ]\n    finally:\n        ds.pm.unregister(name=\"actor_specific_query_plugin\")\n\n\n@pytest.mark.asyncio\nasync def test_actor_endpoint_allows_any_token():\n    ds = Datasette()\n    token = ds.sign(\n        {\n            \"a\": \"root\",\n            \"token\": \"dstok\",\n            \"t\": int(time.time()),\n            \"_r\": {\"a\": [\"debug-menu\"]},\n        },\n        namespace=\"token\",\n    )\n    response = await ds.client.get(\n        \"/-/actor.json\", headers={\"Authorization\": f\"Bearer dstok_{token}\"}\n    )\n    assert response.status_code == 200\n    assert response.json()[\"actor\"] == {\n        \"id\": \"root\",\n        \"token\": \"dstok\",\n        \"_r\": {\"a\": [\"debug-menu\"]},\n    }\n\n\n@pytest.mark.serial\n@pytest.mark.parametrize(\n    \"options,expected\",\n    (\n        ([], {\"id\": \"root\", \"token\": \"dstok\"}),\n        (\n            [\"--all\", \"debug-menu\"],\n            {\"_r\": {\"a\": [\"dm\"]}, \"id\": \"root\", \"token\": \"dstok\"},\n        ),\n        (\n            [\"-a\", \"debug-menu\", \"--all\", \"create-table\"],\n            {\"_r\": {\"a\": [\"dm\", \"ct\"]}, \"id\": \"root\", \"token\": \"dstok\"},\n        ),\n        (\n            [\"-r\", \"db1\", \"t1\", \"insert-row\"],\n            {\"_r\": {\"r\": {\"db1\": {\"t1\": [\"ir\"]}}}, \"id\": \"root\", \"token\": \"dstok\"},\n        ),\n        (\n            [\"-d\", \"db1\", \"create-table\"],\n            {\"_r\": {\"d\": {\"db1\": [\"ct\"]}}, \"id\": \"root\", \"token\": \"dstok\"},\n        ),\n        # And one with all of them multiple times using all the names\n        (\n            [\n                \"-a\",\n                \"debug-menu\",\n                \"--all\",\n                \"create-table\",\n                \"-r\",\n                \"db1\",\n                \"t1\",\n                \"insert-row\",\n                \"--resource\",\n                \"db1\",\n                \"t2\",\n                \"update-row\",\n                \"-d\",\n                \"db1\",\n                \"create-table\",\n                \"--database\",\n                \"db2\",\n                \"drop-table\",\n            ],\n            {\n                \"_r\": {\n                    \"a\": [\"dm\", \"ct\"],\n                    \"d\": {\"db1\": [\"ct\"], \"db2\": [\"dt\"]},\n                    \"r\": {\"db1\": {\"t1\": [\"ir\"], \"t2\": [\"ur\"]}},\n                },\n                \"id\": \"root\",\n                \"token\": \"dstok\",\n            },\n        ),\n    ),\n)\ndef test_cli_create_token(options, expected):\n    runner = CliRunner()\n    result1 = runner.invoke(\n        cli,\n        [\n            \"create-token\",\n            \"--secret\",\n            \"sekrit\",\n            \"root\",\n        ]\n        + options,\n    )\n    token = result1.output.strip()\n    result2 = runner.invoke(\n        cli,\n        [\n            \"serve\",\n            \"--secret\",\n            \"sekrit\",\n            \"--get\",\n            \"/-/actor.json\",\n            \"--token\",\n            token,\n        ],\n    )\n    assert 0 == result2.exit_code, result2.output\n    assert json.loads(result2.output) == {\"actor\": expected}\n\n\n_visible_tables_re = re.compile(r\">\\/((\\w+)\\/(\\w+))\\.json<\\/a> - Get rows for\")\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"is_logged_in,config,expected_visible_tables\",\n    (\n        # Unprotected instance logged out user sees everything:\n        (\n            False,\n            None,\n            [\"perms_ds_one/t1\", \"perms_ds_one/t2\", \"perms_ds_two/t1\"],\n        ),\n        # Fully protected instance logged out user sees nothing\n        (False, {\"allow\": {\"id\": \"user\"}}, None),\n        # User with visibility of just perms_ds_one sees both tables there\n        (\n            True,\n            {\n                \"databases\": {\n                    \"perms_ds_one\": {\"allow\": {\"id\": \"user\"}},\n                    \"perms_ds_two\": {\"allow\": False},\n                }\n            },\n            [\"perms_ds_one/t1\", \"perms_ds_one/t2\"],\n        ),\n        # User with visibility of only table perms_ds_one/t1 sees just that one\n        (\n            True,\n            {\n                \"databases\": {\n                    \"perms_ds_one\": {\n                        \"allow\": {\"id\": \"user\"},\n                        \"tables\": {\"t2\": {\"allow\": False}},\n                    },\n                    \"perms_ds_two\": {\"allow\": False},\n                }\n            },\n            [\"perms_ds_one/t1\"],\n        ),\n    ),\n)\nasync def test_api_explorer_visibility(\n    perms_ds, is_logged_in, config, expected_visible_tables\n):\n    try:\n        prev_config = perms_ds.config\n        perms_ds.config = config or {}\n        cookies = {}\n        if is_logged_in:\n            cookies = {\"ds_actor\": perms_ds.client.actor_cookie({\"id\": \"user\"})}\n        response = await perms_ds.client.get(\"/-/api\", cookies=cookies)\n        if expected_visible_tables:\n            assert response.status_code == 200\n            # Search HTML for stuff matching:\n            # '>/perms_ds_one/t2.json - Get rows for'\n            visible_tables = [\n                match[0] for match in _visible_tables_re.findall(response.text)\n            ]\n            assert visible_tables == expected_visible_tables\n        else:\n            assert response.status_code == 403\n    finally:\n        perms_ds.config = prev_config\n\n\n@pytest.mark.asyncio\nasync def test_view_table_token_cannot_gain_access_without_base_permission(perms_ds):\n    # Only allow a different actor to view this table\n    previous_config = perms_ds.config\n    perms_ds.config = {\n        \"databases\": {\n            \"perms_ds_two\": {\n                # Only someone-else can see anything in this database\n                \"allow\": {\"id\": \"someone-else\"},\n            }\n        }\n    }\n    try:\n        actor = {\n            \"id\": \"restricted-token\",\n            \"token\": \"dstok\",\n            # Restricted token claims access to perms_ds_two/t1 only\n            \"_r\": {\"r\": {\"perms_ds_two\": {\"t1\": [\"vt\"]}}},\n        }\n        cookies = {\"ds_actor\": perms_ds.client.actor_cookie(actor)}\n        response = await perms_ds.client.get(\"/perms_ds_two/t1.json\", cookies=cookies)\n        assert response.status_code == 403\n    finally:\n        perms_ds.config = previous_config\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"restrictions,verb,path,body,expected_status\",\n    (\n        # No restrictions\n        (None, \"get\", \"/.json\", None, 200),\n        (None, \"get\", \"/perms_ds_one.json\", None, 200),\n        (None, \"get\", \"/perms_ds_one/t1.json\", None, 200),\n        (None, \"get\", \"/perms_ds_one/t1/1.json\", None, 200),\n        (None, \"get\", \"/perms_ds_one/v1.json\", None, 200),\n        # Restricted to just view-instance\n        ({\"a\": [\"vi\"]}, \"get\", \"/.json\", None, 200),\n        ({\"a\": [\"vi\"]}, \"get\", \"/perms_ds_one.json\", None, 403),\n        ({\"a\": [\"vi\"]}, \"get\", \"/perms_ds_one/t1.json\", None, 403),\n        ({\"a\": [\"vi\"]}, \"get\", \"/perms_ds_one/t1/1.json\", None, 403),\n        ({\"a\": [\"vi\"]}, \"get\", \"/perms_ds_one/v1.json\", None, 403),\n        # Restricted to just view-database\n        (\n            {\"a\": [\"vd\"]},\n            \"get\",\n            \"/.json\",\n            None,\n            403,\n        ),  # Cannot see instance (no upward cascading)\n        ({\"a\": [\"vd\"]}, \"get\", \"/perms_ds_one.json\", None, 200),\n        ({\"a\": [\"vd\"]}, \"get\", \"/perms_ds_one/t1.json\", None, 403),\n        ({\"a\": [\"vd\"]}, \"get\", \"/perms_ds_one/t1/1.json\", None, 403),\n        ({\"a\": [\"vd\"]}, \"get\", \"/perms_ds_one/v1.json\", None, 403),\n        # Restricted to just view-table for specific database\n        (\n            {\"d\": {\"perms_ds_one\": [\"vt\"]}},\n            \"get\",\n            \"/.json\",\n            None,\n            403,\n        ),  # Cannot see instance (no upward cascading)\n        (\n            {\"d\": {\"perms_ds_one\": [\"vt\"]}},\n            \"get\",\n            \"/perms_ds_one.json\",\n            None,\n            403,\n        ),  # Cannot see database page (no upward cascading)\n        (\n            {\"d\": {\"perms_ds_one\": [\"vt\"]}},\n            \"get\",\n            \"/perms_ds_two.json\",\n            None,\n            403,\n        ),  # But not this one\n        (\n            # Can see the table\n            {\"d\": {\"perms_ds_one\": [\"vt\"]}},\n            \"get\",\n            \"/perms_ds_one/t1.json\",\n            None,\n            200,\n        ),\n        (\n            # And the view\n            {\"d\": {\"perms_ds_one\": [\"vt\"]}},\n            \"get\",\n            \"/perms_ds_one/v1.json\",\n            None,\n            200,\n        ),\n        # view-table access to a specific table\n        (\n            {\"r\": {\"perms_ds_one\": {\"t1\": [\"vt\"]}}},\n            \"get\",\n            \"/.json\",\n            None,\n            403,\n        ),  # Cannot see instance (no upward cascading)\n        (\n            {\"r\": {\"perms_ds_one\": {\"t1\": [\"vt\"]}}},\n            \"get\",\n            \"/perms_ds_one.json\",\n            None,\n            403,\n        ),  # Cannot see database page (no upward cascading)\n        (\n            {\"r\": {\"perms_ds_one\": {\"t1\": [\"vt\"]}}},\n            \"get\",\n            \"/perms_ds_one/t1.json\",\n            None,\n            200,\n        ),\n        # But cannot see the other table\n        (\n            {\"r\": {\"perms_ds_one\": {\"t1\": [\"vt\"]}}},\n            \"get\",\n            \"/perms_ds_one/t2.json\",\n            None,\n            403,\n        ),\n        # Or the view\n        (\n            {\"r\": {\"perms_ds_one\": {\"t1\": [\"vt\"]}}},\n            \"get\",\n            \"/perms_ds_one/v1.json\",\n            None,\n            403,\n        ),\n    ),\n)\nasync def test_actor_restrictions(\n    perms_ds, restrictions, verb, path, body, expected_status\n):\n    actor = {\"id\": \"user\"}\n    if restrictions:\n        actor[\"_r\"] = restrictions\n    method = getattr(perms_ds.client, verb)\n    kwargs = {\"cookies\": {\"ds_actor\": perms_ds.client.actor_cookie(actor)}}\n    if body:\n        kwargs[\"json\"] = body\n    perms_ds._permission_checks.clear()\n    response = await method(path, **kwargs)\n    assert response.status_code == expected_status, json.dumps(\n        {\n            \"verb\": verb,\n            \"path\": path,\n            \"body\": body,\n            \"restrictions\": restrictions,\n            \"expected_status\": expected_status,\n            \"response_status\": response.status_code,\n            \"checks\": [\n                {\n                    \"action\": check.action,\n                    \"parent\": check.parent,\n                    \"child\": check.child,\n                    \"result\": check.result,\n                }\n                for check in perms_ds._permission_checks\n            ],\n        },\n        indent=2,\n    )\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"restrictions,action,resource,expected\",\n    (\n        # Exact match: view-instance restriction allows view-instance action\n        ({\"a\": [\"view-instance\"]}, \"view-instance\", None, True),\n        # No implication: view-table does NOT imply view-instance\n        ({\"a\": [\"view-table\"]}, \"view-instance\", None, False),\n        ({\"a\": [\"view-database\"]}, \"view-instance\", None, False),\n        # update-row does not imply view-instance\n        ({\"a\": [\"update-row\"]}, \"view-instance\", None, False),\n        # view-table on a resource does NOT imply view-instance\n        ({\"r\": {\"db1\": {\"t1\": [\"view-table\"]}}}, \"view-instance\", None, False),\n        # execute-sql on a database does NOT imply view-instance or view-database\n        ({\"d\": {\"db1\": [\"es\"]}}, \"view-instance\", None, False),\n        ({\"d\": {\"db1\": [\"es\"]}}, \"view-database\", \"db1\", False),\n        ({\"d\": {\"db1\": [\"es\"]}}, \"view-database\", \"db2\", False),\n        # But execute-sql abbreviation DOES allow execute-sql action on that database\n        ({\"d\": {\"db1\": [\"es\"]}}, \"execute-sql\", \"db1\", True),\n        # update-row on a resource does not imply view-instance\n        ({\"r\": {\"db1\": {\"t1\": [\"update-row\"]}}}, \"view-instance\", None, False),\n        # view-database on a database does NOT imply view-instance\n        ({\"d\": {\"db1\": [\"view-database\"]}}, \"view-instance\", None, False),\n        # But it DOES allow view-database on that specific database\n        ({\"d\": {\"db1\": [\"view-database\"]}}, \"view-database\", \"db1\", True),\n        # Having view-table on \"a\" allows access to any specific table\n        ({\"a\": [\"view-table\"]}, \"view-table\", (\"dbname\", \"tablename\"), True),\n        # Having view-table on a database allows access to tables in that database\n        (\n            {\"d\": {\"dbname\": [\"view-table\"]}},\n            \"view-table\",\n            (\"dbname\", \"tablename\"),\n            True,\n        ),\n        # But not if it's allowed on a different database\n        (\n            {\"d\": {\"dbname\": [\"view-table\"]}},\n            \"view-table\",\n            (\"dbname2\", \"tablename\"),\n            False,\n        ),\n        # Table-level restriction allows access to that specific table\n        (\n            {\"r\": {\"dbname\": {\"tablename\": [\"view-table\"]}}},\n            \"view-table\",\n            (\"dbname\", \"tablename\"),\n            True,\n        ),\n        # But not to a different table in the same database\n        (\n            {\"r\": {\"dbname\": {\"tablename\": [\"view-table\"]}}},\n            \"view-table\",\n            (\"dbname\", \"other_table\"),\n            False,\n        ),\n    ),\n)\nasync def test_restrictions_allow_action(restrictions, action, resource, expected):\n    ds = Datasette()\n    await ds.invoke_startup()\n    actual = restrictions_allow_action(ds, restrictions, action, resource)\n    assert actual == expected\n\n\n@pytest.mark.asyncio\nasync def test_actor_restrictions_filters_allowed_resources(perms_ds):\n    \"\"\"Test that allowed_resources() respects actor restrictions - issue #2534\"\"\"\n\n    # Actor restricted to just perms_ds_one/t1\n    actor = {\"id\": \"user\", \"_r\": {\"r\": {\"perms_ds_one\": {\"t1\": [\"vt\"]}}}}\n\n    # Should only return t1\n    page = await perms_ds.allowed_resources(\"view-table\", actor)\n    assert len(page.resources) == 1\n    assert page.resources[0].parent == \"perms_ds_one\"\n    assert page.resources[0].child == \"t1\"\n\n    # Database listing should be empty (no view-database permission)\n    db_page = await perms_ds.allowed_resources(\"view-database\", actor)\n    assert len(db_page.resources) == 0\n\n\n@pytest.mark.asyncio\nasync def test_actor_restrictions_do_not_expand_allowed_resources(perms_ds):\n    \"\"\"Restrictions cannot grant access not already allowed to the actor.\"\"\"\n\n    previous_config = perms_ds.config\n    perms_ds.config = {\n        \"databases\": {\n            \"perms_ds_one\": {\n                \"allow\": {\"id\": \"someone-else\"},\n            }\n        }\n    }\n    try:\n        actor = {\"id\": \"user\", \"_r\": {\"r\": {\"perms_ds_one\": {\"t1\": [\"vt\"]}}}}\n\n        # Base actor is not allowed to see t1, so restrictions should not change that\n        page = await perms_ds.allowed_resources(\"view-table\", actor)\n        assert len(page.resources) == 0\n\n        # And explicit permission checks should still deny\n        response = await perms_ds.client.get(\n            \"/perms_ds_one/t1.json\",\n            cookies={\"ds_actor\": perms_ds.client.actor_cookie(actor)},\n        )\n        assert response.status_code == 403\n    finally:\n        perms_ds.config = previous_config\n\n\n@pytest.mark.asyncio\nasync def test_actor_restrictions_database_level(perms_ds):\n    \"\"\"Test database-level restrictions allow all tables in database - issue #2534\"\"\"\n\n    actor = {\"id\": \"user\", \"_r\": {\"d\": {\"perms_ds_one\": [\"vt\"]}}}\n\n    page = await perms_ds.allowed_resources(\"view-table\", actor, parent=\"perms_ds_one\")\n\n    # Should return all tables in perms_ds_one\n    table_names = {r.child for r in page.resources}\n    assert \"t1\" in table_names\n    assert \"t2\" in table_names\n    assert \"v1\" in table_names  # views too\n\n\n@pytest.mark.asyncio\nasync def test_actor_restrictions_global_level(perms_ds):\n    \"\"\"Test global-level restrictions allow all resources - issue #2534\"\"\"\n\n    actor = {\"id\": \"user\", \"_r\": {\"a\": [\"vt\"]}}\n\n    page = await perms_ds.allowed_resources(\"view-table\", actor)\n\n    # Should return all tables in all databases\n    assert len(page.resources) > 0\n    dbs = {r.parent for r in page.resources}\n    assert \"perms_ds_one\" in dbs\n    assert \"perms_ds_two\" in dbs\n\n\n@pytest.mark.asyncio\nasync def test_restrictions_gate_before_config(perms_ds):\n    \"\"\"Test that restrictions act as gating filter before config permissions - issue #2534\"\"\"\n    from datasette.resources import TableResource\n\n    # Actor restricted to just t1 (not t2)\n    actor = {\"id\": \"user\", \"_r\": {\"r\": {\"perms_ds_one\": {\"t1\": [\"vt\"]}}}}\n\n    # Config doesn't matter - restrictions gate what's checked\n    # t2 is not in restriction allowlist, so should be DENIED\n    result = await perms_ds.allowed(\n        action=\"view-table\",\n        resource=TableResource(\"perms_ds_one\", \"t2\"),\n        actor=actor,\n    )\n    assert result is False\n\n    # t1 is in restrictions AND passes normal permission check - should be ALLOWED\n    result = await perms_ds.allowed(\n        action=\"view-table\",\n        resource=TableResource(\"perms_ds_one\", \"t1\"),\n        actor=actor,\n    )\n    assert result is True\n\n\n@pytest.mark.asyncio\nasync def test_actor_restrictions_json_endpoints_show_filtered_listings(perms_ds):\n    \"\"\"Test that /.json and /db.json show correct filtered listings - issue #2534\"\"\"\n\n    actor = {\"id\": \"user\", \"_r\": {\"r\": {\"perms_ds_one\": {\"t1\": [\"vt\"]}}}}\n    cookies = {\"ds_actor\": perms_ds.client.actor_cookie(actor)}\n\n    # /.json should be 403 (no view-instance permission)\n    response = await perms_ds.client.get(\"/.json\", cookies=cookies)\n    assert response.status_code == 403\n\n    # /perms_ds_one.json should be 403 (no view-database permission)\n    response = await perms_ds.client.get(\"/perms_ds_one.json\", cookies=cookies)\n    assert response.status_code == 403\n\n    # /perms_ds_one/t1.json should be 200\n    response = await perms_ds.client.get(\"/perms_ds_one/t1.json\", cookies=cookies)\n    assert response.status_code == 200\n\n\n@pytest.mark.asyncio\nasync def test_actor_restrictions_view_instance_only(perms_ds):\n    \"\"\"Test actor restricted to view-instance only - issue #2534\"\"\"\n\n    actor = {\"id\": \"user\", \"_r\": {\"a\": [\"vi\"]}}\n    cookies = {\"ds_actor\": perms_ds.client.actor_cookie(actor)}\n\n    # /.json should be 200 (has view-instance permission)\n    response = await perms_ds.client.get(\"/.json\", cookies=cookies)\n    assert response.status_code == 200\n\n    # But no databases should be visible (no view-database permission)\n    # The instance is visible but databases list should be empty or minimal\n    # Actually, let's check via allowed_resources\n    page = await perms_ds.allowed_resources(\"view-database\", actor)\n    assert len(page.resources) == 0\n\n\n@pytest.mark.asyncio\nasync def test_actor_restrictions_empty_allowlist(perms_ds):\n    \"\"\"Test actor with empty restrictions allowlist denies everything - issue #2534\"\"\"\n\n    actor = {\"id\": \"user\", \"_r\": {}}\n\n    # No actions in allowlist, so everything should be denied\n    page1 = await perms_ds.allowed_resources(\"view-table\", actor)\n    assert len(page1.resources) == 0\n\n    page2 = await perms_ds.allowed_resources(\"view-database\", actor)\n    assert len(page2.resources) == 0\n\n    result = await perms_ds.allowed(action=\"view-instance\", actor=actor)\n    assert result is False\n\n\n@pytest.mark.asyncio\nasync def test_actor_restrictions_cannot_be_overridden_by_config():\n    \"\"\"Test that config permissions cannot override actor restrictions - issue #2534\"\"\"\n    from datasette.app import Datasette\n    from datasette.resources import TableResource\n\n    # Create datasette with config that allows user to access both t1 AND t2\n    config = {\n        \"databases\": {\n            \"test_db\": {\n                \"tables\": {\n                    \"t1\": {\"allow\": {\"id\": \"user\"}},\n                    \"t2\": {\"allow\": {\"id\": \"user\"}},\n                }\n            }\n        }\n    }\n\n    ds = Datasette(config=config)\n    await ds.invoke_startup()\n    db = ds.add_memory_database(\"test_db\")\n    await db.execute_write(\"create table t1 (id integer primary key)\")\n    await db.execute_write(\"create table t2 (id integer primary key)\")\n\n    # Actor restricted to ONLY t1 (not t2)\n    # Even though config allows t2, restrictions should deny it\n    actor = {\"id\": \"user\", \"_r\": {\"r\": {\"test_db\": {\"t1\": [\"vt\"]}}}}\n\n    # t1 should be allowed (in restrictions AND config allows)\n    result = await ds.allowed(\n        action=\"view-table\", resource=TableResource(\"test_db\", \"t1\"), actor=actor\n    )\n    assert result is True, \"t1 should be allowed - in restriction allowlist\"\n\n    # t2 should be DENIED (not in restrictions, even though config allows)\n    result = await ds.allowed(\n        action=\"view-table\", resource=TableResource(\"test_db\", \"t2\"), actor=actor\n    )\n    assert (\n        result is False\n    ), \"t2 should be denied - NOT in restriction allowlist, config cannot override\"\n\n\n@pytest.mark.asyncio\nasync def test_actor_restrictions_with_database_level_config(perms_ds):\n    \"\"\"Test database-level restrictions with table-level config - issue #2534\"\"\"\n    from datasette.resources import TableResource\n\n    # Config allows specific tables only\n    perms_ds._config = {\n        \"databases\": {\n            \"perms_ds_one\": {\n                \"tables\": {\n                    \"t1\": {\"allow\": {\"id\": \"user\"}},\n                    \"t2\": {\"allow\": {\"id\": \"user\"}},\n                }\n            }\n        }\n    }\n\n    # Actor has database-level restriction (all tables in perms_ds_one)\n    # Should only access tables that pass BOTH restrictions AND config\n    actor = {\"id\": \"user\", \"_r\": {\"d\": {\"perms_ds_one\": [\"vt\"]}}}\n\n    # t1 - in restrictions (all tables) AND config allows\n    result = await perms_ds.allowed(\n        action=\"view-table\", resource=TableResource(\"perms_ds_one\", \"t1\"), actor=actor\n    )\n    assert result is True\n\n    # t2 - in restrictions (all tables) AND config allows\n    result = await perms_ds.allowed(\n        action=\"view-table\", resource=TableResource(\"perms_ds_one\", \"t2\"), actor=actor\n    )\n    assert result is True\n\n    # v1 (view) - in restrictions (all tables) AND config doesn't mention it\n    # Since actor has database-level restriction allowing all tables, v1 is allowed\n    # Config is additive, not restrictive - it doesn't create implicit denies\n    result = await perms_ds.allowed(\n        action=\"view-table\", resource=TableResource(\"perms_ds_one\", \"v1\"), actor=actor\n    )\n    assert result is True, \"v1 should be allowed - actor has db-level restriction\"\n\n    # Clean up\n    perms_ds._config = None\n\n\n@pytest.mark.asyncio\nasync def test_actor_restrictions_parent_deny_blocks_config_child_allow(perms_ds):\n    \"\"\"\n    Test that table-level restrictions add parent-level deny to block\n    other tables in the same database, even if config allows them\n    \"\"\"\n    from datasette.resources import TableResource\n\n    # Config allows both t1 and t2\n    perms_ds._config = {\n        \"databases\": {\n            \"perms_ds_one\": {\n                \"tables\": {\n                    \"t1\": {\"allow\": {\"id\": \"user\"}},\n                    \"t2\": {\"allow\": {\"id\": \"user\"}},\n                }\n            }\n        }\n    }\n\n    # Restriction allows ONLY t1 in perms_ds_one\n    # This should add:\n    # - parent-level DENY for perms_ds_one (to block other tables)\n    # - child-level ALLOW for t1\n    actor = {\"id\": \"user\", \"_r\": {\"r\": {\"perms_ds_one\": {\"t1\": [\"vt\"]}}}}\n\n    # t1 should work (child-level allow beats parent-level deny)\n    result = await perms_ds.allowed(\n        action=\"view-table\", resource=TableResource(\"perms_ds_one\", \"t1\"), actor=actor\n    )\n    assert result is True\n\n    # t2 should be DENIED by parent-level deny from restrictions\n    # even though config has child-level allow\n    # Because restrictions should run first\n    result = await perms_ds.allowed(\n        action=\"view-table\", resource=TableResource(\"perms_ds_one\", \"t2\"), actor=actor\n    )\n    assert (\n        result is False\n    ), \"t2 should be denied - restriction parent deny should beat config child allow\"\n\n    # Clean up\n    perms_ds._config = None\n\n\n@pytest.mark.asyncio\nasync def test_permission_check_view_requires_debug_permission():\n    \"\"\"Test that /-/check requires permissions-debug permission\"\"\"\n    # Anonymous user should be denied\n    ds = Datasette()\n    response = await ds.client.get(\"/-/check.json?action=view-instance\")\n    assert response.status_code == 403\n    assert \"permissions-debug\" in response.text\n\n    # User without permissions-debug should be denied\n    response = await ds.client.get(\n        \"/-/check.json?action=view-instance\",\n        cookies={\"ds_actor\": ds.sign({\"id\": \"user\"}, \"actor\")},\n    )\n    assert response.status_code == 403\n\n    # Root user should have access (root has all permissions)\n    ds_with_root = Datasette()\n    ds_with_root.root_enabled = True\n    root_token = await ds_with_root.create_token(\"root\", handler=\"signed\")\n    response = await ds_with_root.client.get(\n        \"/-/check.json?action=view-instance\",\n        headers={\"Authorization\": f\"Bearer {root_token}\"},\n    )\n    assert response.status_code == 200\n    data = response.json()\n    assert data[\"action\"] == \"view-instance\"\n    assert data[\"allowed\"] is True\n\n\n@pytest.mark.asyncio\nasync def test_root_allow_block_with_table_restricted_actor():\n    \"\"\"\n    Test that root-level allow: blocks are processed for actors with\n    table-level restrictions.\n\n    This covers the case in config.py is_in_restriction_allowlist() where\n    parent=None, child=None and actor has table restrictions but not global.\n    \"\"\"\n    from datasette.resources import TableResource\n\n    # Config with root-level allow block that denies non-admin users\n    ds = Datasette(\n        config={\n            \"allow\": {\"id\": \"admin\"},  # Root-level allow block\n        }\n    )\n    await ds.invoke_startup()\n    db = ds.add_memory_database(\"mydb\")\n    await db.execute_write(\"create table t1 (id integer primary key)\")\n    await ds.client.get(\"/\")  # Trigger catalog refresh\n\n    # Actor with table-level restrictions only (not global)\n    actor = {\"id\": \"user\", \"_r\": {\"r\": {\"mydb\": {\"t1\": [\"view-table\"]}}}}\n\n    # The root-level allow: {id: admin} should be processed and deny this user\n    # because they're not \"admin\", even though they have table restrictions\n    result = await ds.allowed(\n        action=\"view-table\",\n        resource=TableResource(\"mydb\", \"t1\"),\n        actor=actor,\n    )\n    # Should be False because root allow: {id: admin} denies non-admin users\n    assert result is False\n\n    # But admin with same restrictions should be allowed\n    admin_actor = {\"id\": \"admin\", \"_r\": {\"r\": {\"mydb\": {\"t1\": [\"view-table\"]}}}}\n    result = await ds.allowed(\n        action=\"view-table\",\n        resource=TableResource(\"mydb\", \"t1\"),\n        actor=admin_actor,\n    )\n    assert result is True\n"
  },
  {
    "path": "tests/test_plugins.py",
    "content": "from bs4 import BeautifulSoup as Soup\nfrom .fixtures import (\n    make_app_client,\n    TABLES,\n    TEMP_PLUGIN_SECRET_FILE,\n    PLUGINS_DIR,\n    TestClient as _TestClient,\n)  # noqa\nfrom click.testing import CliRunner\nfrom datasette.app import Datasette\nfrom datasette import cli, hookimpl\nfrom datasette.filters import FilterArguments\nfrom datasette.plugins import get_plugins, DEFAULT_PLUGINS, pm\nfrom datasette.permissions import PermissionSQL, Action\nfrom datasette.resources import DatabaseResource\nfrom datasette.utils.sqlite import sqlite3\nfrom datasette.utils import StartupError, await_me_maybe\nfrom jinja2 import ChoiceLoader, FileSystemLoader\nimport base64\nimport datetime\nimport importlib\nimport json\nimport os\nimport pathlib\nimport re\nimport textwrap\nimport pytest\nimport urllib\n\nat_memory_re = re.compile(r\" at 0x\\w+\")\n\n\n@pytest.mark.parametrize(\n    \"plugin_hook\", [name for name in dir(pm.hook) if not name.startswith(\"_\")]\n)\ndef test_plugin_hooks_have_tests(plugin_hook):\n    \"\"\"Every plugin hook should be referenced in this test module\"\"\"\n    tests_in_this_module = [t for t in globals().keys() if t.startswith(\"test_hook_\")]\n    ok = False\n    for test in tests_in_this_module:\n        if plugin_hook in test:\n            ok = True\n    assert ok, f\"Plugin hook is missing tests: {plugin_hook}\"\n\n\n@pytest.mark.asyncio\nasync def test_hook_plugins_dir_plugin_prepare_connection(ds_client):\n    response = await ds_client.get(\n        \"/fixtures/-/query.json?_shape=arrayfirst&sql=select+convert_units(100%2C+'m'%2C+'ft')\"\n    )\n    assert response.json()[0] == pytest.approx(328.0839)\n\n\n@pytest.mark.asyncio\nasync def test_hook_plugin_prepare_connection_arguments(ds_client):\n    response = await ds_client.get(\n        \"/fixtures/-/query.json?sql=select+prepare_connection_args()&_shape=arrayfirst\"\n    )\n    assert [\n        \"database=fixtures, datasette.plugin_config(\\\"name-of-plugin\\\")={'depth': 'root'}\"\n    ] == response.json()\n\n    # Function should not be available on the internal database\n    db = ds_client.ds.get_internal_database()\n    with pytest.raises(sqlite3.OperationalError):\n        await db.execute(\"select prepare_connection_args()\")\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"path,expected_decoded_object\",\n    [\n        (\n            \"/\",\n            {\n                \"template\": \"index.html\",\n                \"database\": None,\n                \"table\": None,\n                \"view_name\": \"index\",\n                \"request_path\": \"/\",\n                \"added\": 15,\n                \"columns\": None,\n            },\n        ),\n        (\n            \"/fixtures\",\n            {\n                \"template\": \"database.html\",\n                \"database\": \"fixtures\",\n                \"table\": None,\n                \"view_name\": \"database\",\n                \"request_path\": \"/fixtures\",\n                \"added\": 15,\n                \"columns\": None,\n            },\n        ),\n        (\n            \"/fixtures/sortable\",\n            {\n                \"template\": \"table.html\",\n                \"database\": \"fixtures\",\n                \"table\": \"sortable\",\n                \"view_name\": \"table\",\n                \"request_path\": \"/fixtures/sortable\",\n                \"added\": 15,\n                \"columns\": [\n                    \"pk1\",\n                    \"pk2\",\n                    \"content\",\n                    \"sortable\",\n                    \"sortable_with_nulls\",\n                    \"sortable_with_nulls_2\",\n                    \"text\",\n                ],\n            },\n        ),\n    ],\n)\nasync def test_hook_extra_css_urls(ds_client, path, expected_decoded_object):\n    response = await ds_client.get(path)\n    assert response.status_code == 200\n    links = Soup(response.text, \"html.parser\").find_all(\"link\")\n    special_href = [\n        link\n        for link in links\n        if link.attrs[\"href\"].endswith(\"/extra-css-urls-demo.css\")\n    ][0][\"href\"]\n    # This link has a base64-encoded JSON blob in it\n    encoded = special_href.split(\"/\")[3]\n    actual_decoded_object = json.loads(base64.b64decode(encoded).decode(\"utf8\"))\n    assert expected_decoded_object == actual_decoded_object\n\n\n@pytest.mark.asyncio\nasync def test_hook_extra_js_urls(ds_client):\n    response = await ds_client.get(\"/\")\n    scripts = Soup(response.text, \"html.parser\").find_all(\"script\")\n    script_attrs = [s.attrs for s in scripts]\n    for attrs in [\n        {\n            \"integrity\": \"SRIHASH\",\n            \"crossorigin\": \"anonymous\",\n            \"src\": \"https://plugin-example.datasette.io/jquery.js\",\n        },\n        {\n            \"src\": \"https://plugin-example.datasette.io/plugin.module.js\",\n            \"type\": \"module\",\n        },\n    ]:\n        assert any(s == attrs for s in script_attrs), \"Expected: {}\".format(attrs)\n\n\n@pytest.mark.asyncio\nasync def test_plugins_with_duplicate_js_urls(ds_client):\n    # If two plugins both require jQuery, jQuery should be loaded only once\n    response = await ds_client.get(\"/fixtures\")\n    # This test is a little tricky, as if the user has any other plugins in\n    # their current virtual environment those may affect what comes back too.\n    # What matters is that https://plugin-example.datasette.io/jquery.js is only there once\n    # and it comes before plugin1.js and plugin2.js which could be in either\n    # order\n    scripts = Soup(response.text, \"html.parser\").find_all(\"script\")\n    srcs = [s[\"src\"] for s in scripts if s.get(\"src\")]\n    # No duplicates allowed:\n    assert len(srcs) == len(set(srcs))\n    # jquery.js loaded once:\n    assert 1 == srcs.count(\"https://plugin-example.datasette.io/jquery.js\")\n    # plugin1.js and plugin2.js are both there:\n    assert 1 == srcs.count(\"https://plugin-example.datasette.io/plugin1.js\")\n    assert 1 == srcs.count(\"https://plugin-example.datasette.io/plugin2.js\")\n    # jquery comes before them both\n    assert srcs.index(\"https://plugin-example.datasette.io/jquery.js\") < srcs.index(\n        \"https://plugin-example.datasette.io/plugin1.js\"\n    )\n    assert srcs.index(\"https://plugin-example.datasette.io/jquery.js\") < srcs.index(\n        \"https://plugin-example.datasette.io/plugin2.js\"\n    )\n\n\n@pytest.mark.asyncio\nasync def test_hook_render_cell_link_from_json(ds_client):\n    sql = \"\"\"\n        select '{\"href\": \"http://example.com/\", \"label\":\"Example\"}'\n    \"\"\".strip()\n    path = \"/fixtures/-/query?\" + urllib.parse.urlencode({\"sql\": sql})\n    response = await ds_client.get(path)\n    td = Soup(response.text, \"html.parser\").find(\"table\").find(\"tbody\").find(\"td\")\n    a = td.find(\"a\")\n    assert a is not None, str(a)\n    assert a.attrs[\"href\"] == \"http://example.com/\"\n    assert a.attrs[\"data-database\"] == \"fixtures\"\n    assert a.text == \"Example\"\n\n\n@pytest.mark.asyncio\nasync def test_hook_render_cell_demo(ds_client):\n    response = await ds_client.get(\n        \"/fixtures/simple_primary_key?id=4&_render_cell_extra=1\"\n    )\n    soup = Soup(response.text, \"html.parser\")\n    td = soup.find(\"td\", {\"class\": \"col-content\"})\n    assert json.loads(td.string) == {\n        \"row\": {\"id\": 4, \"content\": \"RENDER_CELL_DEMO\"},\n        \"column\": \"content\",\n        \"table\": \"simple_primary_key\",\n        \"database\": \"fixtures\",\n        \"pks\": [\"id\"],\n        \"config\": {\"depth\": \"table\", \"special\": \"this-is-simple_primary_key\"},\n        \"render_cell_extra\": 1,\n    }\n\n\n@pytest.mark.asyncio\nasync def test_hook_render_cell_pks_single_pk(ds_client):\n    \"\"\"pks should be [\"id\"] for a table with a single primary key\"\"\"\n    response = await ds_client.get(\"/fixtures/simple_primary_key?id=4\")\n    soup = Soup(response.text, \"html.parser\")\n    td = soup.find(\"td\", {\"class\": \"col-content\"})\n    data = json.loads(td.string)\n    assert data[\"pks\"] == [\"id\"]\n\n\n@pytest.mark.asyncio\nasync def test_hook_render_cell_pks_compound_pk(ds_client):\n    \"\"\"pks should list all primary key columns for a compound pk table\"\"\"\n    response = await ds_client.get(\"/fixtures/compound_primary_key?pk1=d&pk2=e\")\n    soup = Soup(response.text, \"html.parser\")\n    td = soup.find(\"td\", {\"class\": \"col-content\"})\n    data = json.loads(td.string)\n    assert data[\"pks\"] == [\"pk1\", \"pk2\"]\n\n\n@pytest.mark.asyncio\nasync def test_hook_render_cell_pks_rowid_table(ds_client):\n    \"\"\"pks should be [\"rowid\"] for a table with no explicit primary key\"\"\"\n    response = await ds_client.get(\"/fixtures/no_primary_key?content=RENDER_CELL_DEMO\")\n    soup = Soup(response.text, \"html.parser\")\n    td = soup.find(\"td\", {\"class\": \"col-content\"})\n    data = json.loads(td.string)\n    assert data[\"pks\"] == [\"rowid\"]\n\n\n@pytest.mark.asyncio\nasync def test_hook_render_cell_pks_custom_sql(ds_client):\n    \"\"\"pks should be [] for custom SQL queries\"\"\"\n    response = await ds_client.get(\n        \"/fixtures/-/query?sql=select+'RENDER_CELL_DEMO'+as+content\"\n    )\n    soup = Soup(response.text, \"html.parser\")\n    td = soup.find(\"td\", {\"class\": \"col-content\"})\n    data = json.loads(td.string)\n    assert data[\"pks\"] == []\n    assert data[\"table\"] is None\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"path\",\n    (\n        \"/fixtures/-/query?sql=select+'RENDER_CELL_ASYNC'\",\n        \"/fixtures/simple_primary_key\",\n    ),\n)\nasync def test_hook_render_cell_async(ds_client, path):\n    response = await ds_client.get(path)\n    assert b\"RENDER_CELL_ASYNC_RESULT\" in response.content\n\n\n@pytest.mark.asyncio\nasync def test_plugin_config(ds_client):\n    assert {\"depth\": \"table\"} == ds_client.ds.plugin_config(\n        \"name-of-plugin\", database=\"fixtures\", table=\"sortable\"\n    )\n    assert {\"depth\": \"database\"} == ds_client.ds.plugin_config(\n        \"name-of-plugin\", database=\"fixtures\", table=\"unknown_table\"\n    )\n    assert {\"depth\": \"database\"} == ds_client.ds.plugin_config(\n        \"name-of-plugin\", database=\"fixtures\"\n    )\n    assert {\"depth\": \"root\"} == ds_client.ds.plugin_config(\n        \"name-of-plugin\", database=\"unknown_database\"\n    )\n    assert {\"depth\": \"root\"} == ds_client.ds.plugin_config(\"name-of-plugin\")\n    assert None is ds_client.ds.plugin_config(\"unknown-plugin\")\n\n\n@pytest.mark.asyncio\nasync def test_plugin_config_env(ds_client, monkeypatch):\n    monkeypatch.setenv(\"FOO_ENV\", \"FROM_ENVIRONMENT\")\n    assert ds_client.ds.plugin_config(\"env-plugin\") == {\"foo\": \"FROM_ENVIRONMENT\"}\n\n\n@pytest.mark.asyncio\nasync def test_plugin_config_env_from_config(monkeypatch):\n    monkeypatch.setenv(\"FOO_ENV\", \"FROM_ENVIRONMENT_2\")\n    datasette = Datasette(\n        config={\"plugins\": {\"env-plugin\": {\"setting\": {\"$env\": \"FOO_ENV\"}}}}\n    )\n    assert datasette.plugin_config(\"env-plugin\") == {\"setting\": \"FROM_ENVIRONMENT_2\"}\n\n\n@pytest.mark.asyncio\nasync def test_plugin_config_env_from_list(ds_client):\n    os.environ[\"FOO_ENV\"] = \"FROM_ENVIRONMENT\"\n    assert [{\"in_a_list\": \"FROM_ENVIRONMENT\"}] == ds_client.ds.plugin_config(\n        \"env-plugin-list\"\n    )\n    del os.environ[\"FOO_ENV\"]\n\n\n@pytest.mark.asyncio\nasync def test_plugin_config_file(ds_client):\n    with open(TEMP_PLUGIN_SECRET_FILE, \"w\") as fp:\n        fp.write(\"FROM_FILE\")\n    assert {\"foo\": \"FROM_FILE\"} == ds_client.ds.plugin_config(\"file-plugin\")\n    os.remove(TEMP_PLUGIN_SECRET_FILE)\n\n\n@pytest.mark.parametrize(\n    \"path,expected_extra_body_script\",\n    [\n        (\n            \"/\",\n            {\n                \"template\": \"index.html\",\n                \"database\": None,\n                \"table\": None,\n                \"config\": {\"depth\": \"root\"},\n                \"view_name\": \"index\",\n                \"request_path\": \"/\",\n                \"added\": 15,\n                \"columns\": None,\n            },\n        ),\n        (\n            \"/fixtures\",\n            {\n                \"template\": \"database.html\",\n                \"database\": \"fixtures\",\n                \"table\": None,\n                \"config\": {\"depth\": \"database\"},\n                \"view_name\": \"database\",\n                \"request_path\": \"/fixtures\",\n                \"added\": 15,\n                \"columns\": None,\n            },\n        ),\n        (\n            \"/fixtures/sortable\",\n            {\n                \"template\": \"table.html\",\n                \"database\": \"fixtures\",\n                \"table\": \"sortable\",\n                \"config\": {\"depth\": \"table\"},\n                \"view_name\": \"table\",\n                \"request_path\": \"/fixtures/sortable\",\n                \"added\": 15,\n                \"columns\": [\n                    \"pk1\",\n                    \"pk2\",\n                    \"content\",\n                    \"sortable\",\n                    \"sortable_with_nulls\",\n                    \"sortable_with_nulls_2\",\n                    \"text\",\n                ],\n            },\n        ),\n    ],\n)\ndef test_hook_extra_body_script(app_client, path, expected_extra_body_script):\n    r = re.compile(r\"\")\n    response = app_client.get(path)\n    assert response.status_code == 200, response.text\n    match = r.search(response.text)\n    assert match is not None, \"No extra_body_script found in HTML\"\n    json_data = match.group(1)\n    actual_data = json.loads(json_data)\n    assert expected_extra_body_script == actual_data\n\n\n@pytest.mark.asyncio\nasync def test_hook_asgi_wrapper(ds_client):\n    response = await ds_client.get(\"/fixtures\")\n    assert \"fixtures\" == response.headers[\"x-databases\"]\n\n\ndef test_hook_extra_template_vars(restore_working_directory):\n    with make_app_client(\n        template_dir=str(pathlib.Path(__file__).parent / \"test_templates\")\n    ) as client:\n        response = client.get(\"/-/versions\")\n        assert response.status_code == 200\n        extra_template_vars = json.loads(\n            Soup(response.text, \"html.parser\").select(\"pre.extra_template_vars\")[0].text\n        )\n        assert {\n            \"template\": \"show_json.html\",\n            \"scope_path\": \"/-/versions\",\n            \"columns\": None,\n        } == extra_template_vars\n        extra_template_vars_from_awaitable = json.loads(\n            Soup(response.text, \"html.parser\")\n            .select(\"pre.extra_template_vars_from_awaitable\")[0]\n            .text\n        )\n        assert {\n            \"template\": \"show_json.html\",\n            \"awaitable\": True,\n            \"scope_path\": \"/-/versions\",\n        } == extra_template_vars_from_awaitable\n\n\ndef test_plugins_async_template_function(restore_working_directory):\n    with make_app_client(\n        template_dir=str(pathlib.Path(__file__).parent / \"test_templates\")\n    ) as client:\n        response = client.get(\"/-/versions\")\n        assert response.status_code == 200\n        extra_from_awaitable_function = (\n            Soup(response.text, \"html.parser\")\n            .select(\"pre.extra_from_awaitable_function\")[0]\n            .text\n        )\n        expected = (\n            sqlite3.connect(\":memory:\").execute(\"select sqlite_version()\").fetchone()[0]\n        )\n        assert expected == extra_from_awaitable_function\n\n\ndef test_default_plugins_have_no_templates_path_or_static_path():\n    # The default plugins that ship with Datasette should have their static_path and\n    # templates_path all set to None\n    plugins = get_plugins()\n    for plugin in plugins:\n        if plugin[\"name\"] in DEFAULT_PLUGINS:\n            assert None is plugin[\"static_path\"]\n            assert None is plugin[\"templates_path\"]\n\n\n@pytest.fixture(scope=\"session\")\ndef view_names_client(tmp_path_factory):\n    tmpdir = tmp_path_factory.mktemp(\"test-view-names\")\n    templates = tmpdir / \"templates\"\n    templates.mkdir()\n    plugins = tmpdir / \"plugins\"\n    plugins.mkdir()\n    for template in (\n        \"index.html\",\n        \"database.html\",\n        \"table.html\",\n        \"row.html\",\n        \"show_json.html\",\n        \"query.html\",\n    ):\n        (templates / template).write_text(\"view_name:{{ view_name }}\", \"utf-8\")\n    (plugins / \"extra_vars.py\").write_text(\n        textwrap.dedent(\"\"\"\n        from datasette import hookimpl\n        @hookimpl\n        def extra_template_vars(view_name):\n            return {\"view_name\": view_name}\n    \"\"\"),\n        \"utf-8\",\n    )\n    db_path = str(tmpdir / \"fixtures.db\")\n    conn = sqlite3.connect(db_path)\n    conn.executescript(TABLES)\n    return _TestClient(\n        Datasette([db_path], template_dir=str(templates), plugins_dir=str(plugins))\n    )\n\n\n@pytest.mark.parametrize(\n    \"path,view_name\",\n    (\n        (\"/\", \"index\"),\n        (\"/fixtures\", \"database\"),\n        (\"/fixtures/facetable\", \"table\"),\n        (\"/fixtures/facetable/1\", \"row\"),\n        (\"/-/versions\", \"json_data\"),\n        (\"/fixtures/-/query?sql=select+1\", \"database\"),\n    ),\n)\ndef test_view_names(view_names_client, path, view_name):\n    response = view_names_client.get(path)\n    assert response.status_code == 200\n    assert f\"view_name:{view_name}\" == response.text\n\n\n@pytest.mark.asyncio\nasync def test_hook_register_output_renderer_no_parameters(ds_client):\n    response = await ds_client.get(\"/fixtures/facetable.testnone\")\n    assert response.status_code == 200\n    assert b\"Hello\" == response.content\n\n\n@pytest.mark.asyncio\nasync def test_hook_register_output_renderer_all_parameters(ds_client):\n    response = await ds_client.get(\"/fixtures/facetable.testall\")\n    assert response.status_code == 200\n    # Lots of 'at 0x103a4a690' in here - replace those so we can do\n    # an easy comparison\n    body = at_memory_re.sub(\" at 0xXXX\", response.text)\n    assert json.loads(body) == {\n        \"datasette\": \"\",\n        \"columns\": [\n            \"pk\",\n            \"created\",\n            \"planet_int\",\n            \"on_earth\",\n            \"state\",\n            \"_city_id\",\n            \"_neighborhood\",\n            \"tags\",\n            \"complex_array\",\n            \"distinct_some_null\",\n            \"n\",\n        ],\n        \"rows\": [\n            \"\",\n            \"\",\n            \"\",\n            \"\",\n            \"\",\n            \"\",\n            \"\",\n            \"\",\n            \"\",\n            \"\",\n            \"\",\n            \"\",\n            \"\",\n            \"\",\n            \"\",\n        ],\n        \"sql\": \"select pk, created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null, n from facetable order by pk limit 51\",\n        \"query_name\": None,\n        \"database\": \"fixtures\",\n        \"table\": \"facetable\",\n        \"request\": '',\n        \"view_name\": \"table\",\n        \"1+1\": 2,\n    }\n\n\n@pytest.mark.asyncio\nasync def test_hook_register_output_renderer_custom_status_code(ds_client):\n    response = await ds_client.get(\n        \"/fixtures/pragma_cache_size.testall?status_code=202\"\n    )\n    assert response.status_code == 202\n\n\n@pytest.mark.asyncio\nasync def test_hook_register_output_renderer_custom_content_type(ds_client):\n    response = await ds_client.get(\n        \"/fixtures/pragma_cache_size.testall?content_type=text/blah\"\n    )\n    assert \"text/blah\" == response.headers[\"content-type\"]\n\n\n@pytest.mark.asyncio\nasync def test_hook_register_output_renderer_custom_headers(ds_client):\n    response = await ds_client.get(\n        \"/fixtures/pragma_cache_size.testall?header=x-wow:1&header=x-gosh:2\"\n    )\n    assert \"1\" == response.headers[\"x-wow\"]\n    assert \"2\" == response.headers[\"x-gosh\"]\n\n\n@pytest.mark.asyncio\nasync def test_hook_register_output_renderer_returning_response(ds_client):\n    response = await ds_client.get(\"/fixtures/facetable.testresponse\")\n    assert response.status_code == 200\n    assert response.json() == {\"this_is\": \"json\"}\n\n\n@pytest.mark.asyncio\nasync def test_hook_register_output_renderer_returning_broken_value(ds_client):\n    response = await ds_client.get(\"/fixtures/facetable.testresponse?_broken=1\")\n    assert response.status_code == 500\n    assert \"this should break should be dict or Response\" in response.text\n\n\n@pytest.mark.asyncio\nasync def test_hook_register_output_renderer_can_render(ds_client):\n    response = await ds_client.get(\"/fixtures/facetable?_no_can_render=1\")\n    assert response.status_code == 200\n    links = (\n        Soup(response.text, \"html.parser\")\n        .find(\"p\", {\"class\": \"export-links\"})\n        .find_all(\"a\")\n    )\n    actual = [link[\"href\"] for link in links]\n    # Should not be present because we sent ?_no_can_render=1\n    assert \"/fixtures/facetable.testall?_labels=on\" not in actual\n    # Check that it was passed the values we expected\n    assert hasattr(ds_client.ds, \"_can_render_saw\")\n    assert {\n        \"datasette\": ds_client.ds,\n        \"columns\": [\n            \"pk\",\n            \"created\",\n            \"planet_int\",\n            \"on_earth\",\n            \"state\",\n            \"_city_id\",\n            \"_neighborhood\",\n            \"tags\",\n            \"complex_array\",\n            \"distinct_some_null\",\n            \"n\",\n        ],\n        \"sql\": \"select pk, created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null, n from facetable order by pk limit 51\",\n        \"query_name\": None,\n        \"database\": \"fixtures\",\n        \"table\": \"facetable\",\n        \"view_name\": \"table\",\n    }.items() <= ds_client.ds._can_render_saw.items()\n\n\n@pytest.mark.asyncio\nasync def test_hook_prepare_jinja2_environment(ds_client):\n    ds_client.ds._HELLO = \"HI\"\n    await ds_client.ds.invoke_startup()\n    environment = ds_client.ds.get_jinja_environment(None)\n    template = environment.from_string(\n        \"Hello there, {{ a|format_numeric }}, {{ a|to_hello }}, {{ b|select_times_three }}\",\n        {\"a\": 3412341, \"b\": 5},\n    )\n    rendered = await ds_client.ds.render_template(template)\n    assert \"Hello there, 3,412,341, HI, 15\" == rendered\n\n\ndef test_hook_publish_subcommand():\n    # This is hard to test properly, because publish subcommand plugins\n    # cannot be loaded using the --plugins-dir mechanism - they need\n    # to be installed using \"pip install\". So I'm cheating and taking\n    # advantage of the fact that cloudrun/heroku use the plugin hook\n    # to register themselves as default plugins.\n    assert [\"cloudrun\", \"heroku\"] == cli.publish.list_commands({})\n\n\n@pytest.mark.asyncio\nasync def test_hook_register_facet_classes(ds_client):\n    response = await ds_client.get(\n        \"/fixtures/compound_three_primary_keys.json?_dummy_facet=1&_extra=suggested_facets\"\n    )\n    assert response.json()[\"suggested_facets\"] == [\n        {\n            \"name\": \"pk1\",\n            \"toggle_url\": \"http://localhost/fixtures/compound_three_primary_keys.json?_dummy_facet=1&_extra=suggested_facets&_facet_dummy=pk1\",\n            \"type\": \"dummy\",\n        },\n        {\n            \"name\": \"pk2\",\n            \"toggle_url\": \"http://localhost/fixtures/compound_three_primary_keys.json?_dummy_facet=1&_extra=suggested_facets&_facet_dummy=pk2\",\n            \"type\": \"dummy\",\n        },\n        {\n            \"name\": \"pk3\",\n            \"toggle_url\": \"http://localhost/fixtures/compound_three_primary_keys.json?_dummy_facet=1&_extra=suggested_facets&_facet_dummy=pk3\",\n            \"type\": \"dummy\",\n        },\n        {\n            \"name\": \"content\",\n            \"toggle_url\": \"http://localhost/fixtures/compound_three_primary_keys.json?_dummy_facet=1&_extra=suggested_facets&_facet_dummy=content\",\n            \"type\": \"dummy\",\n        },\n        {\n            \"name\": \"pk1\",\n            \"toggle_url\": \"http://localhost/fixtures/compound_three_primary_keys.json?_dummy_facet=1&_extra=suggested_facets&_facet=pk1\",\n        },\n        {\n            \"name\": \"pk2\",\n            \"toggle_url\": \"http://localhost/fixtures/compound_three_primary_keys.json?_dummy_facet=1&_extra=suggested_facets&_facet=pk2\",\n        },\n        {\n            \"name\": \"pk3\",\n            \"toggle_url\": \"http://localhost/fixtures/compound_three_primary_keys.json?_dummy_facet=1&_extra=suggested_facets&_facet=pk3\",\n        },\n    ]\n\n\n@pytest.mark.asyncio\nasync def test_hook_actor_from_request(ds_client):\n    await ds_client.get(\"/\")\n    # Should have no actor\n    assert ds_client.ds._last_request.scope[\"actor\"] is None\n    await ds_client.get(\"/?_bot=1\")\n    # Should have bot actor\n    assert ds_client.ds._last_request.scope[\"actor\"] == {\"id\": \"bot\"}\n\n\n@pytest.mark.asyncio\nasync def test_hook_actor_from_request_async(ds_client):\n    await ds_client.get(\"/\")\n    # Should have no actor\n    assert ds_client.ds._last_request.scope[\"actor\"] is None\n    await ds_client.get(\"/?_bot2=1\")\n    # Should have bot2 actor\n    assert ds_client.ds._last_request.scope[\"actor\"] == {\"id\": \"bot2\", \"1+1\": 2}\n\n\n@pytest.mark.asyncio\nasync def test_existing_scope_actor_respected(ds_client):\n    await ds_client.get(\"/?_actor_in_scope=1\")\n    assert ds_client.ds._last_request.scope[\"actor\"] == {\"id\": \"from-scope\"}\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"action,expected\",\n    [\n        (\"this_is_allowed\", True),\n        (\"this_is_denied\", False),\n        (\"this_is_allowed_async\", True),\n        (\"this_is_denied_async\", False),\n    ],\n)\nasync def test_hook_custom_allowed(action, expected):\n    # Test actions and permission logic are defined in tests/plugins/my_plugin.py\n    ds = Datasette(plugins_dir=PLUGINS_DIR)\n    await ds.invoke_startup()\n    actual = await ds.allowed(action=action, actor={\"id\": \"actor\"})\n    assert expected == actual\n\n\n@pytest.mark.asyncio\nasync def test_hook_permission_resources_sql():\n    ds = Datasette()\n    await ds.invoke_startup()\n\n    collected = []\n    for block in ds.pm.hook.permission_resources_sql(\n        datasette=ds,\n        actor={\"id\": \"alice\"},\n        action=\"view-table\",\n    ):\n        block = await await_me_maybe(block)\n        if block is None:\n            continue\n        if isinstance(block, (list, tuple)):\n            collected.extend(block)\n        else:\n            collected.append(block)\n\n    assert collected\n    assert all(isinstance(item, PermissionSQL) for item in collected)\n\n\n@pytest.mark.asyncio\nasync def test_actor_json(ds_client):\n    assert (await ds_client.get(\"/-/actor.json\")).json() == {\"actor\": None}\n    assert (await ds_client.get(\"/-/actor.json?_bot2=1\")).json() == {\n        \"actor\": {\"id\": \"bot2\", \"1+1\": 2}\n    }\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"path,body\",\n    [\n        (\"/one/\", \"2\"),\n        (\"/two/Ray?greeting=Hail\", \"Hail Ray\"),\n        (\"/not-async/\", \"This was not async\"),\n    ],\n)\nasync def test_hook_register_routes(ds_client, path, body):\n    response = await ds_client.get(path)\n    assert response.status_code == 200\n    assert response.text == body\n\n\n@pytest.mark.parametrize(\"configured_path\", (\"path1\", \"path2\"))\ndef test_hook_register_routes_with_datasette(configured_path):\n    with make_app_client(\n        config={\n            \"plugins\": {\n                \"register-route-demo\": {\n                    \"path\": configured_path,\n                }\n            }\n        }\n    ) as client:\n        response = client.get(f\"/{configured_path}/\")\n        assert response.status_code == 200\n        assert configured_path.upper() == response.text\n        # Other one should 404\n        other_path = [p for p in (\"path1\", \"path2\") if configured_path != p][0]\n        assert client.get(f\"/{other_path}/\", follow_redirects=True).status_code == 404\n\n\ndef test_hook_register_routes_override():\n    \"Plugins can over-ride default paths such as /db/table\"\n    with make_app_client(\n        config={\n            \"plugins\": {\n                \"register-route-demo\": {\n                    \"path\": \"blah\",\n                }\n            }\n        }\n    ) as client:\n        response = client.get(\"/db/table\")\n        assert response.status_code == 200\n        assert (\n            response.text\n            == \"/db/table: [('db_name', 'db'), ('table_and_format', 'table')]\"\n        )\n\n\ndef test_hook_register_routes_post(app_client):\n    response = app_client.post(\"/post/\", {\"this is\": \"post data\"}, csrftoken_from=True)\n    assert response.status_code == 200\n    assert \"csrftoken\" in response.json\n    assert response.json[\"this is\"] == \"post data\"\n\n\ndef test_hook_register_routes_csrftoken(restore_working_directory, tmpdir_factory):\n    templates = tmpdir_factory.mktemp(\"templates\")\n    (templates / \"csrftoken_form.html\").write_text(\n        \"CSRFTOKEN: {{ csrftoken() }}\", \"utf-8\"\n    )\n    with make_app_client(template_dir=templates) as client:\n        response = client.get(\"/csrftoken-form/\")\n        expected_token = client.ds._last_request.scope[\"csrftoken\"]()\n        assert f\"CSRFTOKEN: {expected_token}\" == response.text\n\n\n@pytest.mark.asyncio\nasync def test_hook_register_routes_asgi(ds_client):\n    response = await ds_client.get(\"/three/\")\n    assert {\"hello\": \"world\"} == response.json()\n    assert \"1\" == response.headers[\"x-three\"]\n\n\n@pytest.mark.asyncio\nasync def test_hook_register_routes_add_message(ds_client):\n    response = await ds_client.get(\"/add-message/\")\n    assert response.status_code == 200\n    assert response.text == \"Added message\"\n    decoded = ds_client.ds.unsign(response.cookies[\"ds_messages\"], \"messages\")\n    assert decoded == [[\"Hello from messages\", 1]]\n\n\ndef test_hook_register_routes_render_message(restore_working_directory, tmpdir_factory):\n    templates = tmpdir_factory.mktemp(\"templates\")\n    (templates / \"render_message.html\").write_text('{% extends \"base.html\" %}', \"utf-8\")\n    with make_app_client(template_dir=templates) as client:\n        response1 = client.get(\"/add-message/\")\n        response2 = client.get(\"/render-message/\", cookies=response1.cookies)\n        assert 200 == response2.status\n        assert \"Hello from messages\" in response2.text\n\n\n@pytest.mark.asyncio\nasync def test_hook_startup(ds_client):\n    await ds_client.ds.invoke_startup()\n    assert ds_client.ds._startup_hook_fired\n    assert 2 == ds_client.ds._startup_hook_calculation\n\n\n@pytest.mark.asyncio\nasync def test_hook_startup_metadata_available(ds_client):\n    # Metadata from metadata.yaml should be populated before startup() fires\n    assert \"title\" in ds_client.ds._startup_metadata_keys\n\n\n@pytest.mark.asyncio\nasync def test_hook_startup_catalog_populated(ds_client):\n    # Internal catalog tables should be populated before startup() fires\n    assert \"fixtures\" in ds_client.ds._startup_catalog_databases\n\n\n@pytest.mark.asyncio\nasync def test_hook_canned_queries(ds_client):\n    queries = (await ds_client.get(\"/fixtures.json\")).json()[\"queries\"]\n    queries_by_name = {q[\"name\"]: q for q in queries}\n    assert {\n        \"sql\": \"select 2\",\n        \"name\": \"from_async_hook\",\n        \"private\": False,\n    } == queries_by_name[\"from_async_hook\"]\n    assert {\n        \"sql\": \"select 1, 'null' as actor_id\",\n        \"name\": \"from_hook\",\n        \"private\": False,\n    } == queries_by_name[\"from_hook\"]\n\n\n@pytest.mark.asyncio\nasync def test_hook_canned_queries_non_async(ds_client):\n    response = await ds_client.get(\"/fixtures/from_hook.json?_shape=array\")\n    assert [{\"1\": 1, \"actor_id\": \"null\"}] == response.json()\n\n\n@pytest.mark.asyncio\nasync def test_hook_canned_queries_async(ds_client):\n    response = await ds_client.get(\"/fixtures/from_async_hook.json?_shape=array\")\n    assert [{\"2\": 2}] == response.json()\n\n\n@pytest.mark.asyncio\nasync def test_hook_canned_queries_actor(ds_client):\n    assert (\n        await ds_client.get(\"/fixtures/from_hook.json?_bot=1&_shape=array\")\n    ).json() == [{\"1\": 1, \"actor_id\": \"bot\"}]\n\n\ndef test_hook_register_magic_parameters(restore_working_directory):\n    with make_app_client(\n        extra_databases={\"data.db\": \"create table logs (line text)\"},\n        config={\n            \"databases\": {\n                \"data\": {\n                    \"queries\": {\n                        \"runme\": {\n                            \"sql\": \"insert into logs (line) values (:_request_http_version)\",\n                            \"write\": True,\n                        },\n                        \"get_uuid\": {\n                            \"sql\": \"select :_uuid_new\",\n                        },\n                        \"asyncrequest\": {\n                            \"sql\": \"select :_asyncrequest_key\",\n                        },\n                    }\n                }\n            }\n        },\n    ) as client:\n        response = client.post(\"/data/runme\", {}, csrftoken_from=True)\n        assert response.status_code == 302\n        actual = client.get(\"/data/logs.json?_sort_desc=rowid&_shape=array\").json\n        assert [{\"rowid\": 1, \"line\": \"1.1\"}] == actual\n        # Now try the GET request against get_uuid\n        response_get = client.get(\"/data/get_uuid.json?_shape=array\")\n        assert 200 == response_get.status\n        new_uuid = response_get.json[0][\":_uuid_new\"]\n        assert 4 == new_uuid.count(\"-\")\n        # And test the async one\n        response_async = client.get(\"/data/asyncrequest.json?_shape=array\")\n        assert 200 == response_async.status\n        assert response_async.json[0][\":_asyncrequest_key\"] == \"key\"\n\n\ndef test_hook_forbidden(restore_working_directory):\n    with make_app_client(\n        extra_databases={\"data2.db\": \"create table logs (line text)\"},\n        config={\"allow\": {}},\n    ) as client:\n        response = client.get(\"/\")\n        assert response.status_code == 403\n        response2 = client.get(\"/data2\")\n        assert 302 == response2.status\n        assert (\n            response2.headers[\"Location\"]\n            == \"/login?message=You do not have permission to view this database\"\n        )\n        assert (\n            client.ds._last_forbidden_message\n            == \"You do not have permission to view this database\"\n        )\n\n\n@pytest.mark.asyncio\nasync def test_hook_handle_exception(ds_client):\n    await ds_client.get(\"/trigger-error?x=123\")\n    assert hasattr(ds_client.ds, \"_exception_hook_fired\")\n    request, exception = ds_client.ds._exception_hook_fired\n    assert request.url == \"http://localhost/trigger-error?x=123\"\n    assert isinstance(exception, ZeroDivisionError)\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\"param\", (\"_custom_error\", \"_custom_error_async\"))\nasync def test_hook_handle_exception_custom_response(ds_client, param):\n    response = await ds_client.get(\"/trigger-error?{}=1\".format(param))\n    assert response.text == param\n\n\n@pytest.mark.asyncio\nasync def test_hook_menu_links(ds_client):\n    def get_menu_links(html):\n        soup = Soup(html, \"html.parser\")\n        return [\n            {\"label\": a.text, \"href\": a[\"href\"]} for a in soup.select(\".nav-menu a\")\n        ]\n\n    response = await ds_client.get(\"/\")\n    assert get_menu_links(response.text) == []\n\n    response_2 = await ds_client.get(\"/?_bot=1&_hello=BOB\")\n    assert get_menu_links(response_2.text) == [\n        {\"label\": \"Hello, BOB\", \"href\": \"/\"},\n        {\"label\": \"Hello 2\", \"href\": \"/\"},\n    ]\n\n\n@pytest.mark.asyncio\nasync def test_hook_table_actions(ds_client):\n    response = await ds_client.get(\"/fixtures/facetable\")\n    assert get_actions_links(response.text) == []\n    response_2 = await ds_client.get(\"/fixtures/facetable?_bot=1&_hello=BOB\")\n    assert \">Table actions<\" in response_2.text\n    assert sorted(\n        get_actions_links(response_2.text), key=lambda link: link[\"label\"]\n    ) == [\n        {\"label\": \"Database: fixtures\", \"href\": \"/\", \"description\": None},\n        {\"label\": \"From async BOB\", \"href\": \"/\", \"description\": None},\n        {\"label\": \"Table: facetable\", \"href\": \"/\", \"description\": None},\n    ]\n\n\n@pytest.mark.asyncio\nasync def test_hook_view_actions(ds_client):\n    response = await ds_client.get(\"/fixtures/simple_view\")\n    assert get_actions_links(response.text) == []\n    response_2 = await ds_client.get(\n        \"/fixtures/simple_view\",\n        cookies={\"ds_actor\": ds_client.actor_cookie({\"id\": \"bob\"})},\n    )\n    assert \">View actions<\" in response_2.text\n    assert sorted(\n        get_actions_links(response_2.text), key=lambda link: link[\"label\"]\n    ) == [\n        {\"label\": \"Database: fixtures\", \"href\": \"/\", \"description\": None},\n        {\"label\": \"View: simple_view\", \"href\": \"/\", \"description\": None},\n    ]\n\n\ndef get_actions_links(html):\n    soup = Soup(html, \"html.parser\")\n    details = soup.find(\"details\", {\"class\": \"actions-menu-links\"})\n    if details is None:\n        return []\n    links = []\n    for a_el in details.select(\"a\"):\n        description = None\n        if a_el.find(\"p\") is not None:\n            description = a_el.find(\"p\").text.strip()\n            a_el.find(\"p\").extract()\n        label = a_el.text.strip()\n        href = a_el[\"href\"]\n        links.append({\"label\": label, \"href\": href, \"description\": description})\n    return links\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"path,expected_url\",\n    (\n        (\"/fixtures/-/query?sql=select+1\", \"/fixtures/-/query?sql=explain+select+1\"),\n        pytest.param(\n            \"/fixtures/pragma_cache_size\",\n            \"/fixtures/-/query?sql=explain+PRAGMA+cache_size%3B\",\n        ),\n        # Don't attempt to explain an explain\n        (\"/fixtures/-/query?sql=explain+select+1\", None),\n    ),\n)\nasync def test_hook_query_actions(ds_client, path, expected_url):\n    response = await ds_client.get(path)\n    assert response.status_code == 200\n    links = get_actions_links(response.text)\n    if expected_url is None:\n        assert links == []\n    else:\n        assert links == [\n            {\n                \"label\": \"Explain this query\",\n                \"href\": expected_url,\n                \"description\": \"Runs a SQLite explain\",\n            }\n        ]\n\n\n@pytest.mark.asyncio\nasync def test_hook_row_actions(ds_client):\n    response = await ds_client.get(\"/fixtures/facet_cities/1\")\n    assert get_actions_links(response.text) == []\n\n    response_2 = await ds_client.get(\n        \"/fixtures/facet_cities/1\",\n        cookies={\"ds_actor\": ds_client.actor_cookie({\"id\": \"sam\"})},\n    )\n    assert get_actions_links(response_2.text) == [\n        {\n            \"label\": \"Row details for sam\",\n            \"href\": \"/\",\n            \"description\": '{\"id\": 1, \"name\": \"San Francisco\"}',\n        }\n    ]\n\n\n@pytest.mark.asyncio\nasync def test_hook_database_actions(ds_client):\n    response = await ds_client.get(\"/fixtures\")\n    assert get_actions_links(response.text) == []\n\n    response_2 = await ds_client.get(\"/fixtures?_bot=1&_hello=BOB\")\n    assert get_actions_links(response_2.text) == [\n        {\"label\": \"Database: fixtures - BOB\", \"href\": \"/\", \"description\": None},\n    ]\n\n\n@pytest.mark.asyncio\nasync def test_hook_homepage_actions(ds_client):\n    response = await ds_client.get(\"/\")\n    # No button for anonymous users\n    assert \"Homepage actions\" not in response.text\n    # Signed in user gets an action\n    response2 = await ds_client.get(\n        \"/\", cookies={\"ds_actor\": ds_client.actor_cookie({\"id\": \"troy\"})}\n    )\n    assert \"Homepage actions\" in response2.text\n    assert get_actions_links(response2.text) == [\n        {\n            \"label\": \"Custom homepage for: troy\",\n            \"href\": \"/-/custom-homepage\",\n            \"description\": None,\n        },\n    ]\n\n\ndef test_hook_skip_csrf(app_client):\n    cookie = app_client.actor_cookie({\"id\": \"test\"})\n    csrf_response = app_client.post(\n        \"/post/\",\n        post_data={\"this is\": \"post data\"},\n        csrftoken_from=True,\n        cookies={\"ds_actor\": cookie},\n    )\n    assert csrf_response.status_code == 200\n    missing_csrf_response = app_client.post(\n        \"/post/\", post_data={\"this is\": \"post data\"}, cookies={\"ds_actor\": cookie}\n    )\n    assert missing_csrf_response.status_code == 403\n    # But \"/skip-csrf\" should allow\n    allow_csrf_response = app_client.post(\n        \"/skip-csrf\", post_data={\"this is\": \"post data\"}, cookies={\"ds_actor\": cookie}\n    )\n    assert allow_csrf_response.status_code == 405  # Method not allowed\n    # /skip-csrf-2 should not\n    second_missing_csrf_response = app_client.post(\n        \"/skip-csrf-2\", post_data={\"this is\": \"post data\"}, cookies={\"ds_actor\": cookie}\n    )\n    assert second_missing_csrf_response.status_code == 403\n\n\ndef _extract_commands(output):\n    lines = output.split(\"Commands:\\n\", 1)[1].split(\"\\n\")\n    return {line.split()[0].replace(\"*\", \"\") for line in lines if line.strip()}\n\n\ndef test_hook_register_commands():\n    # Without the plugin should have seven commands\n    runner = CliRunner()\n    result = runner.invoke(cli.cli, \"--help\")\n    commands = _extract_commands(result.output)\n    assert commands == {\n        \"serve\",\n        \"inspect\",\n        \"install\",\n        \"package\",\n        \"plugins\",\n        \"publish\",\n        \"uninstall\",\n        \"create-token\",\n    }\n\n    # Now install a plugin\n    class VerifyPlugin:\n        __name__ = \"VerifyPlugin\"\n\n        @hookimpl\n        def register_commands(self, cli):\n            @cli.command()\n            def verify():\n                pass\n\n            @cli.command()\n            def unverify():\n                pass\n\n    pm.register(VerifyPlugin(), name=\"verify\")\n    importlib.reload(cli)\n    result2 = runner.invoke(cli.cli, \"--help\")\n    commands2 = _extract_commands(result2.output)\n    assert commands2 == {\n        \"serve\",\n        \"inspect\",\n        \"install\",\n        \"package\",\n        \"plugins\",\n        \"publish\",\n        \"uninstall\",\n        \"verify\",\n        \"unverify\",\n        \"create-token\",\n    }\n    pm.unregister(name=\"verify\")\n    importlib.reload(cli)\n\n\n@pytest.mark.asyncio\nasync def test_hook_filters_from_request(ds_client):\n    class ReturnNothingPlugin:\n        __name__ = \"ReturnNothingPlugin\"\n\n        @hookimpl\n        def filters_from_request(self, request):\n            if request.args.get(\"_nothing\"):\n                return FilterArguments([\"1 = 0\"], human_descriptions=[\"NOTHING\"])\n\n    ds_client.ds.pm.register(ReturnNothingPlugin(), name=\"ReturnNothingPlugin\")\n    response = await ds_client.get(\"/fixtures/facetable?_nothing=1\")\n    assert \"0 rows\\n        where NOTHING\" in response.text\n    json_response = await ds_client.get(\"/fixtures/facetable.json?_nothing=1\")\n    assert json_response.json()[\"rows\"] == []\n    ds_client.ds.pm.unregister(name=\"ReturnNothingPlugin\")\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\"extra_metadata\", (False, True))\nasync def test_hook_register_actions(extra_metadata):\n\n    ds = Datasette(\n        config=(\n            {\n                \"plugins\": {\n                    \"datasette-register-actions\": {\n                        \"actions\": [\n                            {\n                                \"name\": \"extra-from-metadata\",\n                                \"abbr\": \"efm\",\n                                \"description\": \"Extra from metadata\",\n                            }\n                        ]\n                    }\n                }\n            }\n            if extra_metadata\n            else None\n        ),\n        plugins_dir=PLUGINS_DIR,\n    )\n    await ds.invoke_startup()\n    assert ds.actions[\"action-from-plugin\"] == Action(\n        name=\"action-from-plugin\",\n        abbr=\"ap\",\n        description=\"New action added by a plugin\",\n        resource_class=DatabaseResource,\n    )\n    if extra_metadata:\n        assert ds.actions[\"extra-from-metadata\"] == Action(\n            name=\"extra-from-metadata\",\n            abbr=\"efm\",\n            description=\"Extra from metadata\",\n        )\n    else:\n        assert \"extra-from-metadata\" not in ds.actions\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\"duplicate\", (\"name\", \"abbr\"))\nasync def test_hook_register_actions_no_duplicates(duplicate):\n    name1, name2 = \"name1\", \"name2\"\n    abbr1, abbr2 = \"abbr1\", \"abbr2\"\n    if duplicate == \"name\":\n        name2 = \"name1\"\n    if duplicate == \"abbr\":\n        abbr2 = \"abbr1\"\n    ds = Datasette(\n        config={\n            \"plugins\": {\n                \"datasette-register-actions\": {\n                    \"actions\": [\n                        {\n                            \"name\": name1,\n                            \"abbr\": abbr1,\n                            \"description\": None,\n                        },\n                        {\n                            \"name\": name2,\n                            \"abbr\": abbr2,\n                            \"description\": None,\n                        },\n                    ]\n                }\n            }\n        },\n        plugins_dir=PLUGINS_DIR,\n    )\n    # This should error:\n    with pytest.raises(StartupError) as ex:\n        await ds.invoke_startup()\n        assert \"Duplicate action {}\".format(duplicate) in str(ex.value)\n\n\n@pytest.mark.asyncio\nasync def test_hook_register_actions_allows_identical_duplicates():\n    ds = Datasette(\n        config={\n            \"plugins\": {\n                \"datasette-register-actions\": {\n                    \"actions\": [\n                        {\n                            \"name\": \"name1\",\n                            \"abbr\": \"abbr1\",\n                            \"description\": None,\n                        },\n                        {\n                            \"name\": \"name1\",\n                            \"abbr\": \"abbr1\",\n                            \"description\": None,\n                        },\n                    ]\n                }\n            }\n        },\n        plugins_dir=PLUGINS_DIR,\n    )\n    await ds.invoke_startup()\n    # Check that ds.actions has only one of each\n    assert len([p for p in ds.actions.values() if p.abbr == \"abbr1\"]) == 1\n\n\n@pytest.mark.asyncio\nasync def test_hook_actors_from_ids():\n    # Without the hook should return default {\"id\": id} list\n    ds = Datasette()\n    await ds.invoke_startup()\n    db = ds.add_memory_database(\"actors_from_ids\")\n    await db.execute_write(\n        \"create table actors (id text primary key, name text, age int)\"\n    )\n    await db.execute_write(\n        \"insert into actors (id, name, age) values ('3', 'Cate Blanchett', 52)\"\n    )\n    await db.execute_write(\n        \"insert into actors (id, name, age) values ('5', 'Rooney Mara', 36)\"\n    )\n    await db.execute_write(\n        \"insert into actors (id, name, age) values ('7', 'Sarah Paulson', 46)\"\n    )\n    await db.execute_write(\n        \"insert into actors (id, name, age) values ('9', 'Helena Bonham Carter', 55)\"\n    )\n    table_names = await db.table_names()\n    assert table_names == [\"actors\"]\n    actors1 = await ds.actors_from_ids([\"3\", \"5\", \"7\"])\n    assert actors1 == {\n        \"3\": {\"id\": \"3\"},\n        \"5\": {\"id\": \"5\"},\n        \"7\": {\"id\": \"7\"},\n    }\n\n    class ActorsFromIdsPlugin:\n        __name__ = \"ActorsFromIdsPlugin\"\n\n        @hookimpl\n        def actors_from_ids(self, datasette, actor_ids):\n            db = datasette.get_database(\"actors_from_ids\")\n\n            async def inner():\n                sql = \"select id, name from actors where id in ({})\".format(\n                    \", \".join(\"?\" for _ in actor_ids)\n                )\n                actors = {}\n                result = await db.execute(sql, actor_ids)\n                for row in result.rows:\n                    actor = dict(row)\n                    actors[actor[\"id\"]] = actor\n                return actors\n\n            return inner\n\n    try:\n        ds.pm.register(ActorsFromIdsPlugin(), name=\"ActorsFromIdsPlugin\")\n        actors2 = await ds.actors_from_ids([\"3\", \"5\", \"7\"])\n        assert actors2 == {\n            \"3\": {\"id\": \"3\", \"name\": \"Cate Blanchett\"},\n            \"5\": {\"id\": \"5\", \"name\": \"Rooney Mara\"},\n            \"7\": {\"id\": \"7\", \"name\": \"Sarah Paulson\"},\n        }\n    finally:\n        ds.pm.unregister(name=\"ReturnNothingPlugin\")\n\n\n@pytest.mark.asyncio\nasync def test_plugin_is_installed():\n    datasette = Datasette(memory=True)\n\n    class DummyPlugin:\n        __name__ = \"DummyPlugin\"\n\n        @hookimpl\n        def actors_from_ids(self, datasette, actor_ids):\n            return {}\n\n    try:\n        datasette.pm.register(DummyPlugin(), name=\"DummyPlugin\")\n        response = await datasette.client.get(\"/-/plugins.json\")\n        assert response.status_code == 200\n        installed_plugins = {p[\"name\"] for p in response.json()}\n        assert \"DummyPlugin\" in installed_plugins\n\n    finally:\n        datasette.pm.unregister(name=\"DummyPlugin\")\n\n\n@pytest.mark.asyncio\nasync def test_hook_jinja2_environment_from_request(tmpdir):\n    templates = pathlib.Path(tmpdir / \"templates\")\n    templates.mkdir()\n    (templates / \"index.html\").write_text(\"Hello museums!\", \"utf-8\")\n\n    class EnvironmentPlugin:\n        @hookimpl\n        def jinja2_environment_from_request(self, request, env):\n            if request and request.host == \"www.niche-museums.com\":\n                return env.overlay(\n                    loader=ChoiceLoader(\n                        [\n                            FileSystemLoader(str(templates)),\n                            env.loader,\n                        ]\n                    ),\n                    enable_async=True,\n                )\n            return env\n\n    datasette = Datasette(memory=True)\n\n    try:\n        datasette.pm.register(EnvironmentPlugin(), name=\"EnvironmentPlugin\")\n        response = await datasette.client.get(\"/\")\n        assert response.status_code == 200\n        assert \"Hello museums!\" not in response.text\n        # Try again with the hostname\n        response2 = await datasette.client.get(\n            \"/\", headers={\"host\": \"www.niche-museums.com\"}\n        )\n        assert response2.status_code == 200\n        assert \"Hello museums!\" in response2.text\n    finally:\n        datasette.pm.unregister(name=\"EnvironmentPlugin\")\n\n\nclass SlotPlugin:\n    __name__ = \"SlotPlugin\"\n\n    @hookimpl\n    def top_homepage(self, request):\n        return \"Xtop_homepage:\" + request.args[\"z\"]\n\n    @hookimpl\n    def top_database(self, request, database):\n        async def inner():\n            return \"Xtop_database:{}:{}\".format(database, request.args[\"z\"])\n\n        return inner\n\n    @hookimpl\n    def top_table(self, request, database, table):\n        return \"Xtop_table:{}:{}:{}\".format(database, table, request.args[\"z\"])\n\n    @hookimpl\n    def top_row(self, request, database, table, row):\n        return \"Xtop_row:{}:{}:{}:{}\".format(\n            database, table, row[\"name\"], request.args[\"z\"]\n        )\n\n    @hookimpl\n    def top_query(self, request, database, sql):\n        return \"Xtop_query:{}:{}:{}\".format(database, sql, request.args[\"z\"])\n\n    @hookimpl\n    def top_canned_query(self, request, database, query_name):\n        return \"Xtop_query:{}:{}:{}\".format(database, query_name, request.args[\"z\"])\n\n\n@pytest.mark.asyncio\nasync def test_hook_top_homepage():\n    datasette = Datasette(memory=True)\n    try:\n        datasette.pm.register(SlotPlugin(), name=\"SlotPlugin\")\n        response = await datasette.client.get(\"/?z=foo\")\n        assert response.status_code == 200\n        assert \"Xtop_homepage:foo\" in response.text\n    finally:\n        datasette.pm.unregister(name=\"SlotPlugin\")\n\n\n@pytest.mark.asyncio\nasync def test_hook_top_database():\n    datasette = Datasette(memory=True)\n    try:\n        datasette.pm.register(SlotPlugin(), name=\"SlotPlugin\")\n        response = await datasette.client.get(\"/_memory?z=bar\")\n        assert response.status_code == 200\n        assert \"Xtop_database:_memory:bar\" in response.text\n    finally:\n        datasette.pm.unregister(name=\"SlotPlugin\")\n\n\n@pytest.mark.asyncio\nasync def test_hook_top_table(ds_client):\n    try:\n        ds_client.ds.pm.register(SlotPlugin(), name=\"SlotPlugin\")\n        response = await ds_client.get(\"/fixtures/facetable?z=baz\")\n        assert response.status_code == 200\n        assert \"Xtop_table:fixtures:facetable:baz\" in response.text\n    finally:\n        ds_client.ds.pm.unregister(name=\"SlotPlugin\")\n\n\n@pytest.mark.asyncio\nasync def test_hook_top_row(ds_client):\n    try:\n        ds_client.ds.pm.register(SlotPlugin(), name=\"SlotPlugin\")\n        response = await ds_client.get(\"/fixtures/facet_cities/1?z=bax\")\n        assert response.status_code == 200\n        assert \"Xtop_row:fixtures:facet_cities:San Francisco:bax\" in response.text\n    finally:\n        ds_client.ds.pm.unregister(name=\"SlotPlugin\")\n\n\n@pytest.mark.asyncio\nasync def test_hook_top_query(ds_client):\n    try:\n        pm.register(SlotPlugin(), name=\"SlotPlugin\")\n        response = await ds_client.get(\"/fixtures/-/query?sql=select+1&z=x\")\n        assert response.status_code == 200\n        assert \"Xtop_query:fixtures:select 1:x\" in response.text\n    finally:\n        pm.unregister(name=\"SlotPlugin\")\n\n\n@pytest.mark.asyncio\nasync def test_hook_top_canned_query(ds_client):\n    try:\n        pm.register(SlotPlugin(), name=\"SlotPlugin\")\n        response = await ds_client.get(\"/fixtures/from_hook?z=xyz\")\n        assert response.status_code == 200\n        assert \"Xtop_query:fixtures:from_hook:xyz\" in response.text\n    finally:\n        pm.unregister(name=\"SlotPlugin\")\n\n\n@pytest.mark.asyncio\nasync def test_hook_track_event():\n    datasette = Datasette(memory=True)\n    from .conftest import TrackEventPlugin\n\n    await datasette.invoke_startup()\n    await datasette.track_event(\n        TrackEventPlugin.OneEvent(actor=None, extra=\"extra extra\")\n    )\n    assert len(datasette._tracked_events) == 1\n    assert isinstance(datasette._tracked_events[0], TrackEventPlugin.OneEvent)\n    event = datasette._tracked_events[0]\n    assert event.name == \"one\"\n    assert event.properties() == {\"extra\": \"extra extra\"}\n    # Should have a recent created as well\n    created = event.created\n    assert isinstance(created, datetime.datetime)\n    assert created.tzinfo == datetime.timezone.utc\n\n\n@pytest.mark.asyncio\nasync def test_hook_register_events():\n    datasette = Datasette(memory=True)\n    await datasette.invoke_startup()\n    assert any(k.__name__ == \"OneEvent\" for k in datasette.event_classes)\n\n\n@pytest.mark.asyncio\nasync def test_hook_register_token_handler(ds_client):\n    handlers = ds_client.ds._token_handlers()\n    handler_names = [h.name for h in handlers]\n    # Both the default signed handler and the test hardcoded handler\n    assert \"signed\" in handler_names\n    assert \"hardcoded\" in handler_names\n\n    # Create a token using the hardcoded handler (first registered from plugins dir)\n    token = await ds_client.ds.create_token(\"test-user\")\n    assert token.startswith(\"dstok_hardcoded_token_\")\n\n    # Verify it\n    actor = await ds_client.ds.verify_token(token)\n    assert actor[\"id\"] == \"hardcoded-actor\"\n    assert actor[\"token\"] == \"hardcoded\"\n\n    # Create a token by explicitly requesting the hardcoded handler by name\n    token2 = await ds_client.ds.create_token(\"test-user\", handler=\"hardcoded\")\n    assert token2.startswith(\"dstok_hardcoded_token_\")\n    actor2 = await ds_client.ds.verify_token(token2)\n    assert actor2[\"id\"] == \"hardcoded-actor\"\n\n    # Create a token by explicitly requesting the signed handler by name\n    signed_token = await ds_client.ds.create_token(\"test-user\", handler=\"signed\")\n    assert signed_token.startswith(\"dstok_\")\n    assert not signed_token.startswith(\"dstok_hardcoded_token_\")\n    signed_actor = await ds_client.ds.verify_token(signed_token)\n    assert signed_actor[\"id\"] == \"test-user\"\n    assert signed_actor[\"token\"] == \"dstok\"\n\n\n@pytest.mark.asyncio\nasync def test_hook_write_wrapper():\n    datasette = Datasette(memory=True)\n    log = []\n\n    class WrapWritePlugin:\n        __name__ = \"WrapWritePlugin\"\n\n        @staticmethod\n        @hookimpl\n        def write_wrapper(datasette, database, request, transaction):\n            if database != \"_memory\":\n                return None\n\n            def wrapper(conn):\n                log.append(\"before\")\n                yield\n                log.append(\"after\")\n\n            return wrapper\n\n    pm.register(WrapWritePlugin(), name=\"WrapWritePluginTest\")\n    try:\n        db = datasette.get_database(\"_memory\")\n        await db.execute_write(\"create table t (id integer primary key)\")\n        assert log == [\"before\", \"after\"]\n    finally:\n        pm.unregister(name=\"WrapWritePluginTest\")\n\n\n@pytest.mark.asyncio\nasync def test_hook_register_actions_view_collection():\n    datasette = Datasette(memory=True, plugins_dir=PLUGINS_DIR)\n    await datasette.invoke_startup()\n    # Check that the custom action from my_plugin.py is registered\n    assert \"view-collection\" in datasette.actions\n    action = datasette.actions[\"view-collection\"]\n    assert action.abbr == \"vc\"\n    assert action.description == \"View a collection\"\n\n\n@pytest.mark.asyncio\nasync def test_hook_register_actions_with_custom_resources():\n    \"\"\"\n    Test registering actions with custom Resource classes:\n    - A global action (no resource)\n    - A parent-level action (DocumentCollectionResource)\n    - A child-level action (DocumentResource)\n    \"\"\"\n    from datasette.permissions import Resource\n\n    # Define custom Resource classes\n    class DocumentCollectionResource(Resource):\n        \"\"\"A collection of documents.\"\"\"\n\n        name = \"document_collection\"\n        parent_class = None  # Top-level resource\n\n        def __init__(self, collection: str):\n            super().__init__(parent=collection, child=None)\n\n        @classmethod\n        async def resources_sql(cls, datasette, actor=None) -> str:\n            return \"\"\"\n                SELECT 'collection1' AS parent, NULL AS child\n                UNION ALL\n                SELECT 'collection2' AS parent, NULL AS child\n            \"\"\"\n\n    class DocumentResource(Resource):\n        \"\"\"A document in a collection.\"\"\"\n\n        name = \"document\"\n        parent_class = DocumentCollectionResource  # Child of DocumentCollectionResource\n\n        def __init__(self, collection: str, document: str):\n            super().__init__(parent=collection, child=document)\n\n        @classmethod\n        async def resources_sql(cls, datasette, actor=None) -> str:\n            return \"\"\"\n                SELECT 'collection1' AS parent, 'doc1' AS child\n                UNION ALL\n                SELECT 'collection1' AS parent, 'doc2' AS child\n                UNION ALL\n                SELECT 'collection2' AS parent, 'doc3' AS child\n            \"\"\"\n\n    # Define a test plugin that registers these actions\n    class TestPlugin:\n        __name__ = \"test_custom_resources_plugin\"\n\n        @hookimpl\n        def register_actions(self, datasette):\n            return [\n                # Global action - no resource_class\n                Action(\n                    name=\"manage-documents\",\n                    abbr=\"md\",\n                    description=\"Manage the document system\",\n                ),\n                # Parent-level action - collection only\n                Action(\n                    name=\"view-document-collection\",\n                    description=\"View a document collection\",\n                    resource_class=DocumentCollectionResource,\n                ),\n                # Child-level action - collection + document\n                Action(\n                    name=\"view-document\",\n                    abbr=\"vdoc\",\n                    description=\"View a document\",\n                    resource_class=DocumentResource,\n                ),\n            ]\n\n        @hookimpl\n        def permission_resources_sql(self, datasette, actor, action):\n            from datasette.permissions import PermissionSQL\n\n            # Grant user2 access to manage-documents globally\n            if actor and actor.get(\"id\") == \"user2\" and action == \"manage-documents\":\n                return PermissionSQL.allow(reason=\"user2 granted manage-documents\")\n\n            # Grant user2 access to view-document-collection globally\n            if (\n                actor\n                and actor.get(\"id\") == \"user2\"\n                and action == \"view-document-collection\"\n            ):\n                return PermissionSQL.allow(\n                    reason=\"user2 granted view-document-collection\"\n                )\n\n            # Default allow for view-document-collection (like other view-* actions)\n            if action == \"view-document-collection\":\n                return PermissionSQL.allow(\n                    reason=\"default allow for view-document-collection\"\n                )\n\n            # Default allow for view-document (like other view-* actions)\n            if action == \"view-document\":\n                return PermissionSQL.allow(reason=\"default allow for view-document\")\n\n    # Register the plugin temporarily\n    plugin = TestPlugin()\n    pm.register(plugin, name=\"test_custom_resources_plugin\")\n\n    try:\n        # Create datasette instance and invoke startup\n        datasette = Datasette(memory=True)\n        await datasette.invoke_startup()\n\n        # Test global action\n        manage_docs = datasette.actions[\"manage-documents\"]\n        assert manage_docs.name == \"manage-documents\"\n        assert manage_docs.abbr == \"md\"\n        assert manage_docs.resource_class is None\n        assert manage_docs.takes_parent is False\n        assert manage_docs.takes_child is False\n\n        # Test parent-level action\n        view_collection = datasette.actions[\"view-document-collection\"]\n        assert view_collection.name == \"view-document-collection\"\n        assert view_collection.abbr is None\n        assert view_collection.resource_class is DocumentCollectionResource\n        assert view_collection.takes_parent is True\n        assert view_collection.takes_child is False\n\n        # Test child-level action\n        view_doc = datasette.actions[\"view-document\"]\n        assert view_doc.name == \"view-document\"\n        assert view_doc.abbr == \"vdoc\"\n        assert view_doc.resource_class is DocumentResource\n        assert view_doc.takes_parent is True\n        assert view_doc.takes_child is True\n\n        # Verify the resource classes have correct hierarchy\n        assert DocumentCollectionResource.parent_class is None\n        assert DocumentResource.parent_class is DocumentCollectionResource\n\n        # Test that resources can be instantiated correctly\n        collection_resource = DocumentCollectionResource(collection=\"collection1\")\n        assert collection_resource.parent == \"collection1\"\n        assert collection_resource.child is None\n\n        doc_resource = DocumentResource(collection=\"collection1\", document=\"doc1\")\n        assert doc_resource.parent == \"collection1\"\n        assert doc_resource.child == \"doc1\"\n\n        # Test permission checks with restricted actors\n\n        # Test 1: Global action - no restrictions (custom actions default to deny)\n        unrestricted_actor = {\"id\": \"user1\"}\n        allowed = await datasette.allowed(\n            action=\"manage-documents\",\n            actor=unrestricted_actor,\n        )\n        assert allowed is False  # Custom actions have no default allow\n\n        # Test 2: Global action - user2 has explicit permission via plugin hook\n        restricted_global = {\"id\": \"user2\", \"_r\": {\"a\": [\"md\"]}}\n        allowed = await datasette.allowed(\n            action=\"manage-documents\",\n            actor=restricted_global,\n        )\n        assert allowed is True  # Granted by plugin hook for user2\n\n        # Test 3: Global action - restricted but not in allowlist\n        restricted_no_access = {\"id\": \"user3\", \"_r\": {\"a\": [\"vdc\"]}}\n        allowed = await datasette.allowed(\n            action=\"manage-documents\",\n            actor=restricted_no_access,\n        )\n        assert allowed is False  # Not in allowlist\n\n        # Test 4: Collection-level action - allowed for specific collection\n        collection_resource = DocumentCollectionResource(collection=\"collection1\")\n        # This one does not have an abbreviation:\n        restricted_collection = {\n            \"id\": \"user4\",\n            \"_r\": {\"d\": {\"collection1\": [\"view-document-collection\"]}},\n        }\n        allowed = await datasette.allowed(\n            action=\"view-document-collection\",\n            resource=collection_resource,\n            actor=restricted_collection,\n        )\n        assert allowed is True  # Allowed for collection1\n\n        # Test 5: Collection-level action - denied for different collection\n        collection2_resource = DocumentCollectionResource(collection=\"collection2\")\n        allowed = await datasette.allowed(\n            action=\"view-document-collection\",\n            resource=collection2_resource,\n            actor=restricted_collection,\n        )\n        assert allowed is False  # Not allowed for collection2\n\n        # Test 6: Document-level action - allowed for specific document\n        doc1_resource = DocumentResource(collection=\"collection1\", document=\"doc1\")\n        restricted_document = {\n            \"id\": \"user5\",\n            \"_r\": {\"r\": {\"collection1\": {\"doc1\": [\"vdoc\"]}}},\n        }\n        allowed = await datasette.allowed(\n            action=\"view-document\",\n            resource=doc1_resource,\n            actor=restricted_document,\n        )\n        assert allowed is True  # Allowed for collection1/doc1\n\n        # Test 7: Document-level action - denied for different document\n        doc2_resource = DocumentResource(collection=\"collection1\", document=\"doc2\")\n        allowed = await datasette.allowed(\n            action=\"view-document\",\n            resource=doc2_resource,\n            actor=restricted_document,\n        )\n        assert allowed is False  # Not allowed for collection1/doc2\n\n        # Test 8: Document-level action - globally allowed\n        doc_resource = DocumentResource(collection=\"collection2\", document=\"doc3\")\n        restricted_all_docs = {\"id\": \"user6\", \"_r\": {\"a\": [\"vdoc\"]}}\n        allowed = await datasette.allowed(\n            action=\"view-document\",\n            resource=doc_resource,\n            actor=restricted_all_docs,\n        )\n        assert allowed is True  # Globally allowed for all documents\n\n        # Test 9: Verify hierarchy - collection access doesn't grant document access\n        collection_only_actor = {\"id\": \"user7\", \"_r\": {\"d\": {\"collection1\": [\"vdc\"]}}}\n        doc_resource = DocumentResource(collection=\"collection1\", document=\"doc1\")\n        allowed = await datasette.allowed(\n            action=\"view-document\",\n            resource=doc_resource,\n            actor=collection_only_actor,\n        )\n        assert (\n            allowed is False\n        )  # Collection permission doesn't grant document permission\n\n    finally:\n        # Unregister the plugin\n        pm.unregister(plugin)\n\n\n@pytest.mark.skip(reason=\"TODO\")\n@pytest.mark.parametrize(\n    \"metadata,config,expected_metadata,expected_config\",\n    (\n        (\n            # Instance level\n            {\"plugins\": {\"datasette-foo\": \"bar\"}},\n            {},\n            {},\n            {\"plugins\": {\"datasette-foo\": \"bar\"}},\n        ),\n        (\n            # Database level\n            {\"databases\": {\"foo\": {\"plugins\": {\"datasette-foo\": \"bar\"}}}},\n            {},\n            {},\n            {\"databases\": {\"foo\": {\"plugins\": {\"datasette-foo\": \"bar\"}}}},\n        ),\n        (\n            # Table level\n            {\n                \"databases\": {\n                    \"foo\": {\"tables\": {\"bar\": {\"plugins\": {\"datasette-foo\": \"bar\"}}}}\n                }\n            },\n            {},\n            {},\n            {\n                \"databases\": {\n                    \"foo\": {\"tables\": {\"bar\": {\"plugins\": {\"datasette-foo\": \"bar\"}}}}\n                }\n            },\n        ),\n        (\n            # Keep other keys\n            {\"plugins\": {\"datasette-foo\": \"bar\"}, \"other\": \"key\"},\n            {\"original_config\": \"original\"},\n            {\"other\": \"key\"},\n            {\"original_config\": \"original\", \"plugins\": {\"datasette-foo\": \"bar\"}},\n        ),\n    ),\n)\ndef test_metadata_plugin_config_treated_as_config(\n    metadata, config, expected_metadata, expected_config\n):\n    ds = Datasette(metadata=metadata, config=config)\n    actual_metadata = ds.metadata()\n    assert \"plugins\" not in actual_metadata\n    assert actual_metadata == expected_metadata\n    assert ds.config == expected_config\n\n\n@pytest.mark.asyncio\nasync def test_hook_register_column_types():\n    ds = Datasette()\n    await ds.invoke_startup()\n    # Built-in column types should be registered\n    assert \"url\" in ds._column_types\n    assert \"email\" in ds._column_types\n    assert \"json\" in ds._column_types\n    assert \"nonexistent\" not in ds._column_types\n"
  },
  {
    "path": "tests/test_publish_cloudrun.py",
    "content": "from click.testing import CliRunner\nfrom datasette import cli\nfrom unittest import mock\nimport json\nimport os\nimport pytest\nimport textwrap\n\n\n@pytest.mark.serial\n@mock.patch(\"shutil.which\")\ndef test_publish_cloudrun_requires_gcloud(mock_which, tmp_path_factory):\n    mock_which.return_value = False\n    runner = CliRunner()\n    os.chdir(tmp_path_factory.mktemp(\"runner\"))\n    with open(\"test.db\", \"w\") as fp:\n        fp.write(\"data\")\n    result = runner.invoke(cli.cli, [\"publish\", \"cloudrun\", \"test.db\"])\n    assert result.exit_code == 1\n    assert \"Publishing to Google Cloud requires gcloud\" in result.output\n\n\n@mock.patch(\"shutil.which\")\ndef test_publish_cloudrun_invalid_database(mock_which):\n    mock_which.return_value = True\n    runner = CliRunner()\n    result = runner.invoke(cli.cli, [\"publish\", \"cloudrun\", \"woop.db\"])\n    assert result.exit_code == 2\n    assert \"Path 'woop.db' does not exist\" in result.output\n\n\n@pytest.mark.serial\n@mock.patch(\"shutil.which\")\n@mock.patch(\"datasette.publish.cloudrun.check_output\")\n@mock.patch(\"datasette.publish.cloudrun.check_call\")\n@mock.patch(\"datasette.publish.cloudrun.get_existing_services\")\ndef test_publish_cloudrun_prompts_for_service(\n    mock_get_existing_services, mock_call, mock_output, mock_which, tmp_path_factory\n):\n    mock_get_existing_services.return_value = [\n        {\"name\": \"existing\", \"created\": \"2019-01-01\", \"url\": \"http://www.example.com/\"}\n    ]\n    mock_output.return_value = \"myproject\"\n    mock_which.return_value = True\n    runner = CliRunner()\n    os.chdir(tmp_path_factory.mktemp(\"runner\"))\n    with open(\"test.db\", \"w\") as fp:\n        fp.write(\"data\")\n    result = runner.invoke(\n        cli.cli, [\"publish\", \"cloudrun\", \"test.db\"], input=\"input-service\"\n    )\n    assert (\n        \"Please provide a service name for this deployment\\n\\n\"\n        \"Using an existing service name will over-write it\\n\\n\"\n        \"Your existing services:\\n\\n\"\n        \"  existing - created 2019-01-01 - http://www.example.com/\\n\\n\"\n        \"Service name: input-service\"\n    ) == result.output.strip()\n    assert 0 == result.exit_code\n    tag = \"us-docker.pkg.dev/myproject/datasette/datasette-input-service\"\n    mock_call.assert_has_calls(\n        [\n            mock.call(\n                \"gcloud services enable artifactregistry.googleapis.com --project myproject --quiet\",\n                shell=True,\n            ),\n            mock.call(\n                \"gcloud artifacts repositories describe datasette --project myproject --location us --quiet\",\n                shell=True,\n            ),\n            mock.call(f\"gcloud builds submit --tag {tag}\", shell=True),\n            mock.call(\n                \"gcloud run deploy --allow-unauthenticated --platform=managed --image {} input-service --max-instances 1\".format(\n                    tag\n                ),\n                shell=True,\n            ),\n        ]\n    )\n\n\n@pytest.mark.serial\n@mock.patch(\"shutil.which\")\n@mock.patch(\"datasette.publish.cloudrun.check_output\")\n@mock.patch(\"datasette.publish.cloudrun.check_call\")\ndef test_publish_cloudrun(mock_call, mock_output, mock_which, tmp_path_factory):\n    mock_output.return_value = \"myproject\"\n    mock_which.return_value = True\n    runner = CliRunner()\n    os.chdir(tmp_path_factory.mktemp(\"runner\"))\n    with open(\"test.db\", \"w\") as fp:\n        fp.write(\"data\")\n    result = runner.invoke(\n        cli.cli, [\"publish\", \"cloudrun\", \"test.db\", \"--service\", \"test\"]\n    )\n    assert 0 == result.exit_code\n    tag = f\"us-docker.pkg.dev/{mock_output.return_value}/datasette/datasette-test\"\n    mock_call.assert_has_calls(\n        [\n            mock.call(\n                f\"gcloud services enable artifactregistry.googleapis.com --project {mock_output.return_value} --quiet\",\n                shell=True,\n            ),\n            mock.call(\n                f\"gcloud artifacts repositories describe datasette --project {mock_output.return_value} --location us --quiet\",\n                shell=True,\n            ),\n            mock.call(f\"gcloud builds submit --tag {tag}\", shell=True),\n            mock.call(\n                \"gcloud run deploy --allow-unauthenticated --platform=managed --image {} test --max-instances 1\".format(\n                    tag\n                ),\n                shell=True,\n            ),\n        ]\n    )\n\n\n@pytest.mark.serial\n@mock.patch(\"shutil.which\")\n@mock.patch(\"datasette.publish.cloudrun.check_output\")\n@mock.patch(\"datasette.publish.cloudrun.check_call\")\n@pytest.mark.parametrize(\n    \"memory,cpu,timeout,min_instances,max_instances,expected_gcloud_args\",\n    [\n        [\"1Gi\", None, None, None, None, \"--memory 1Gi\"],\n        [\"2G\", None, None, None, None, \"--memory 2G\"],\n        [\"256Mi\", None, None, None, None, \"--memory 256Mi\"],\n        [\n            \"4\",\n            None,\n            None,\n            None,\n            None,\n            None,\n        ],\n        [\n            \"GB\",\n            None,\n            None,\n            None,\n            None,\n            None,\n        ],\n        [None, 1, None, None, None, \"--cpu 1\"],\n        [None, 2, None, None, None, \"--cpu 2\"],\n        [None, 3, None, None, None, None],\n        [None, 4, None, None, None, \"--cpu 4\"],\n        [\"2G\", 4, None, None, None, \"--memory 2G --cpu 4\"],\n        [None, None, 1800, None, None, \"--timeout 1800\"],\n        [None, None, None, 2, None, \"--min-instances 2\"],\n        [None, None, None, 2, 4, \"--min-instances 2 --max-instances 4\"],\n        [None, 2, None, None, 4, \"--cpu 2 --max-instances 4\"],\n    ],\n)\ndef test_publish_cloudrun_memory_cpu(\n    mock_call,\n    mock_output,\n    mock_which,\n    memory,\n    cpu,\n    timeout,\n    min_instances,\n    max_instances,\n    expected_gcloud_args,\n    tmp_path_factory,\n):\n    mock_output.return_value = \"myproject\"\n    mock_which.return_value = True\n    runner = CliRunner()\n    os.chdir(tmp_path_factory.mktemp(\"runner\"))\n    with open(\"test.db\", \"w\") as fp:\n        fp.write(\"data\")\n    args = [\"publish\", \"cloudrun\", \"test.db\", \"--service\", \"test\"]\n    if memory:\n        args.extend([\"--memory\", memory])\n    if cpu:\n        args.extend([\"--cpu\", str(cpu)])\n    if timeout:\n        args.extend([\"--timeout\", str(timeout)])\n    result = runner.invoke(cli.cli, args)\n    if expected_gcloud_args is None:\n        assert 2 == result.exit_code\n        return\n    assert 0 == result.exit_code\n    tag = f\"us-docker.pkg.dev/{mock_output.return_value}/datasette/datasette-test\"\n    expected_call = (\n        \"gcloud run deploy --allow-unauthenticated --platform=managed\"\n        \" --image {} test\".format(tag)\n    )\n    expected_build_call = f\"gcloud builds submit --tag {tag}\"\n    if memory:\n        expected_call += \" --memory {}\".format(memory)\n    if cpu:\n        expected_call += \" --cpu {}\".format(cpu)\n    if timeout:\n        expected_build_call += f\" --timeout {timeout}\"\n    # max_instances defaults to 1\n    expected_call += \" --max-instances 1\"\n    mock_call.assert_has_calls(\n        [\n            mock.call(\n                f\"gcloud services enable artifactregistry.googleapis.com --project {mock_output.return_value} --quiet\",\n                shell=True,\n            ),\n            mock.call(\n                f\"gcloud artifacts repositories describe datasette --project {mock_output.return_value} --location us --quiet\",\n                shell=True,\n            ),\n            mock.call(expected_build_call, shell=True),\n            mock.call(\n                expected_call,\n                shell=True,\n            ),\n        ]\n    )\n\n\n@pytest.mark.serial\n@mock.patch(\"shutil.which\")\n@mock.patch(\"datasette.publish.cloudrun.check_output\")\n@mock.patch(\"datasette.publish.cloudrun.check_call\")\ndef test_publish_cloudrun_plugin_secrets(\n    mock_call, mock_output, mock_which, tmp_path_factory\n):\n    mock_which.return_value = True\n    mock_output.return_value = \"myproject\"\n\n    runner = CliRunner()\n    os.chdir(tmp_path_factory.mktemp(\"runner\"))\n    with open(\"test.db\", \"w\") as fp:\n        fp.write(\"data\")\n    with open(\"metadata.yml\", \"w\") as fp:\n        fp.write(textwrap.dedent(\"\"\"\n            title: Hello from metadata YAML\n            plugins:\n              datasette-auth-github:\n                foo: bar\n            \"\"\").strip())\n    result = runner.invoke(\n        cli.cli,\n        [\n            \"publish\",\n            \"cloudrun\",\n            \"test.db\",\n            \"--metadata\",\n            \"metadata.yml\",\n            \"--service\",\n            \"datasette\",\n            \"--plugin-secret\",\n            \"datasette-auth-github\",\n            \"client_id\",\n            \"x-client-id\",\n            \"--show-files\",\n            \"--secret\",\n            \"x-secret\",\n        ],\n    )\n    assert result.exit_code == 0\n    dockerfile = (\n        result.output.split(\"==== Dockerfile ====\\n\")[1]\n        .split(\"\\n====================\\n\")[0]\n        .strip()\n    )\n    expected = textwrap.dedent(\n        r\"\"\"\n    FROM python:3.11.0-slim-bullseye\n    COPY . /app\n    WORKDIR /app\n\n    ENV DATASETTE_AUTH_GITHUB_CLIENT_ID 'x-client-id'\n    ENV DATASETTE_SECRET 'x-secret'\n    RUN pip install -U datasette\n    RUN datasette inspect test.db --inspect-file inspect-data.json\n    ENV PORT 8001\n    EXPOSE 8001\n    CMD datasette serve --host 0.0.0.0 -i test.db --cors --inspect-file inspect-data.json --metadata metadata.json --setting force_https_urls on --port $PORT\"\"\"\n    ).strip()\n    assert expected == dockerfile\n    metadata = (\n        result.output.split(\"=== metadata.json ===\\n\")[1]\n        .split(\"\\n==== Dockerfile ====\\n\")[0]\n        .strip()\n    )\n    assert {\n        \"title\": \"Hello from metadata YAML\",\n        \"plugins\": {\n            \"datasette-auth-github\": {\n                \"client_id\": {\"$env\": \"DATASETTE_AUTH_GITHUB_CLIENT_ID\"},\n                \"foo\": \"bar\",\n            },\n        },\n    } == json.loads(metadata)\n\n\n@pytest.mark.serial\n@mock.patch(\"shutil.which\")\n@mock.patch(\"datasette.publish.cloudrun.check_output\")\n@mock.patch(\"datasette.publish.cloudrun.check_call\")\ndef test_publish_cloudrun_apt_get_install(\n    mock_call, mock_output, mock_which, tmp_path_factory\n):\n    mock_which.return_value = True\n    mock_output.return_value = \"myproject\"\n\n    runner = CliRunner()\n    os.chdir(tmp_path_factory.mktemp(\"runner\"))\n    with open(\"test.db\", \"w\") as fp:\n        fp.write(\"data\")\n    result = runner.invoke(\n        cli.cli,\n        [\n            \"publish\",\n            \"cloudrun\",\n            \"test.db\",\n            \"--service\",\n            \"datasette\",\n            \"--show-files\",\n            \"--secret\",\n            \"x-secret\",\n            \"--apt-get-install\",\n            \"ripgrep\",\n            \"--spatialite\",\n        ],\n    )\n    assert result.exit_code == 0\n    dockerfile = (\n        result.output.split(\"==== Dockerfile ====\\n\")[1]\n        .split(\"\\n====================\\n\")[0]\n        .strip()\n    )\n    expected = textwrap.dedent(r\"\"\"\n    FROM python:3.11.0-slim-bullseye\n    COPY . /app\n    WORKDIR /app\n\n    RUN apt-get update && \\\n        apt-get install -y ripgrep python3-dev gcc libsqlite3-mod-spatialite && \\\n        rm -rf /var/lib/apt/lists/*\n\n    ENV DATASETTE_SECRET 'x-secret'\n    ENV SQLITE_EXTENSIONS '/usr/lib/x86_64-linux-gnu/mod_spatialite.so'\n    RUN pip install -U datasette\n    RUN datasette inspect test.db --inspect-file inspect-data.json\n    ENV PORT 8001\n    EXPOSE 8001\n    CMD datasette serve --host 0.0.0.0 -i test.db --cors --inspect-file inspect-data.json --setting force_https_urls on --port $PORT\n    \"\"\").strip()\n    assert expected == dockerfile\n\n\n@pytest.mark.serial\n@mock.patch(\"shutil.which\")\n@mock.patch(\"datasette.publish.cloudrun.check_output\")\n@mock.patch(\"datasette.publish.cloudrun.check_call\")\n@pytest.mark.parametrize(\n    \"extra_options,expected\",\n    [\n        (\"\", \"--setting force_https_urls on\"),\n        (\n            \"--setting base_url /foo\",\n            \"--setting base_url /foo --setting force_https_urls on\",\n        ),\n        (\"--setting force_https_urls off\", \"--setting force_https_urls off\"),\n    ],\n)\ndef test_publish_cloudrun_extra_options(\n    mock_call, mock_output, mock_which, extra_options, expected, tmp_path_factory\n):\n    mock_which.return_value = True\n    mock_output.return_value = \"myproject\"\n\n    runner = CliRunner()\n    os.chdir(tmp_path_factory.mktemp(\"runner\"))\n    with open(\"test.db\", \"w\") as fp:\n        fp.write(\"data\")\n    result = runner.invoke(\n        cli.cli,\n        [\n            \"publish\",\n            \"cloudrun\",\n            \"test.db\",\n            \"--service\",\n            \"datasette\",\n            \"--show-files\",\n            \"--extra-options\",\n            extra_options,\n        ],\n    )\n    assert result.exit_code == 0\n    dockerfile = (\n        result.output.split(\"==== Dockerfile ====\\n\")[1]\n        .split(\"\\n====================\\n\")[0]\n        .strip()\n    )\n    last_line = dockerfile.split(\"\\n\")[-1]\n    extra_options = (\n        last_line.split(\"--inspect-file inspect-data.json\")[1]\n        .split(\"--port\")[0]\n        .strip()\n    )\n    assert extra_options == expected\n"
  },
  {
    "path": "tests/test_publish_heroku.py",
    "content": "from click.testing import CliRunner\nfrom datasette import cli\nfrom unittest import mock\nimport os\nimport pathlib\nimport pytest\n\n\n@pytest.mark.serial\n@mock.patch(\"shutil.which\")\ndef test_publish_heroku_requires_heroku(mock_which, tmp_path_factory):\n    mock_which.return_value = False\n    runner = CliRunner()\n    os.chdir(tmp_path_factory.mktemp(\"runner\"))\n    with open(\"test.db\", \"w\") as fp:\n        fp.write(\"data\")\n    result = runner.invoke(cli.cli, [\"publish\", \"heroku\", \"test.db\"])\n    assert result.exit_code == 1\n    assert \"Publishing to Heroku requires heroku\" in result.output\n\n\n@pytest.mark.serial\n@mock.patch(\"shutil.which\")\n@mock.patch(\"datasette.publish.heroku.check_output\")\n@mock.patch(\"datasette.publish.heroku.call\")\ndef test_publish_heroku_installs_plugin(\n    mock_call, mock_check_output, mock_which, tmp_path_factory\n):\n    mock_which.return_value = True\n    mock_check_output.side_effect = lambda s: {\"['heroku', 'plugins']\": b\"\"}[repr(s)]\n    runner = CliRunner()\n    os.chdir(tmp_path_factory.mktemp(\"runner\"))\n    with open(\"t.db\", \"w\") as fp:\n        fp.write(\"data\")\n    result = runner.invoke(cli.cli, [\"publish\", \"heroku\", \"t.db\"], input=\"y\\n\")\n    assert 0 != result.exit_code\n    mock_check_output.assert_has_calls(\n        [mock.call([\"heroku\", \"plugins\"]), mock.call([\"heroku\", \"apps:list\", \"--json\"])]\n    )\n    mock_call.assert_has_calls(\n        [mock.call([\"heroku\", \"plugins:install\", \"heroku-builds\"])]\n    )\n\n\n@mock.patch(\"shutil.which\")\ndef test_publish_heroku_invalid_database(mock_which):\n    mock_which.return_value = True\n    runner = CliRunner()\n    result = runner.invoke(cli.cli, [\"publish\", \"heroku\", \"woop.db\"])\n    assert result.exit_code == 2\n    assert \"Path 'woop.db' does not exist\" in result.output\n\n\n@pytest.mark.serial\n@mock.patch(\"shutil.which\")\n@mock.patch(\"datasette.publish.heroku.check_output\")\n@mock.patch(\"datasette.publish.heroku.call\")\ndef test_publish_heroku(mock_call, mock_check_output, mock_which, tmp_path_factory):\n    mock_which.return_value = True\n    mock_check_output.side_effect = lambda s: {\n        \"['heroku', 'plugins']\": b\"heroku-builds\",\n        \"['heroku', 'apps:list', '--json']\": b\"[]\",\n        \"['heroku', 'apps:create', 'datasette', '--json']\": b'{\"name\": \"f\"}',\n    }[repr(s)]\n    runner = CliRunner()\n    os.chdir(tmp_path_factory.mktemp(\"runner\"))\n    with open(\"test.db\", \"w\") as fp:\n        fp.write(\"data\")\n    result = runner.invoke(cli.cli, [\"publish\", \"heroku\", \"test.db\", \"--tar\", \"gtar\"])\n    assert 0 == result.exit_code, result.output\n    mock_call.assert_has_calls(\n        [\n            mock.call(\n                [\n                    \"heroku\",\n                    \"builds:create\",\n                    \"-a\",\n                    \"f\",\n                    \"--include-vcs-ignore\",\n                    \"--tar\",\n                    \"gtar\",\n                ]\n            ),\n        ]\n    )\n\n\n@pytest.mark.serial\n@mock.patch(\"shutil.which\")\n@mock.patch(\"datasette.publish.heroku.check_output\")\n@mock.patch(\"datasette.publish.heroku.call\")\ndef test_publish_heroku_plugin_secrets(\n    mock_call, mock_check_output, mock_which, tmp_path_factory\n):\n    mock_which.return_value = True\n    mock_check_output.side_effect = lambda s: {\n        \"['heroku', 'plugins']\": b\"heroku-builds\",\n        \"['heroku', 'apps:list', '--json']\": b\"[]\",\n        \"['heroku', 'apps:create', 'datasette', '--json']\": b'{\"name\": \"f\"}',\n    }[repr(s)]\n    runner = CliRunner()\n    os.chdir(tmp_path_factory.mktemp(\"runner\"))\n    with open(\"test.db\", \"w\") as fp:\n        fp.write(\"data\")\n    result = runner.invoke(\n        cli.cli,\n        [\n            \"publish\",\n            \"heroku\",\n            \"test.db\",\n            \"--plugin-secret\",\n            \"datasette-auth-github\",\n            \"client_id\",\n            \"x-client-id\",\n        ],\n    )\n    assert 0 == result.exit_code, result.output\n    mock_call.assert_has_calls(\n        [\n            mock.call(\n                [\n                    \"heroku\",\n                    \"config:set\",\n                    \"-a\",\n                    \"f\",\n                    \"DATASETTE_AUTH_GITHUB_CLIENT_ID=x-client-id\",\n                ]\n            ),\n            mock.call([\"heroku\", \"builds:create\", \"-a\", \"f\", \"--include-vcs-ignore\"]),\n        ]\n    )\n\n\n@pytest.mark.serial\n@mock.patch(\"shutil.which\")\n@mock.patch(\"datasette.publish.heroku.check_output\")\n@mock.patch(\"datasette.publish.heroku.call\")\ndef test_publish_heroku_generate_dir(\n    mock_call, mock_check_output, mock_which, tmp_path_factory\n):\n    mock_which.return_value = True\n    mock_check_output.side_effect = lambda s: {\n        \"['heroku', 'plugins']\": b\"heroku-builds\",\n    }[repr(s)]\n    runner = CliRunner()\n    os.chdir(tmp_path_factory.mktemp(\"runner\"))\n    with open(\"test.db\", \"w\") as fp:\n        fp.write(\"data\")\n    output = str(tmp_path_factory.mktemp(\"generate_dir\") / \"output\")\n    result = runner.invoke(\n        cli.cli,\n        [\n            \"publish\",\n            \"heroku\",\n            \"test.db\",\n            \"--generate-dir\",\n            output,\n        ],\n    )\n    assert result.exit_code == 0\n    path = pathlib.Path(output)\n    assert path.exists()\n    file_names = {str(r.relative_to(path)) for r in path.glob(\"*\")}\n    assert file_names == {\n        \"requirements.txt\",\n        \"bin\",\n        \"runtime.txt\",\n        \"Procfile\",\n        \"test.db\",\n    }\n    for name, expected in (\n        (\"requirements.txt\", \"datasette\"),\n        (\"runtime.txt\", \"python-3.11.0\"),\n        (\n            \"Procfile\",\n            (\n                \"web: datasette serve --host 0.0.0.0 -i test.db \"\n                \"--cors --port $PORT --inspect-file inspect-data.json\"\n            ),\n        ),\n    ):\n        with open(path / name) as fp:\n            assert fp.read().strip() == expected\n"
  },
  {
    "path": "tests/test_restriction_sql.py",
    "content": "import pytest\nfrom datasette.app import Datasette\nfrom datasette.permissions import PermissionSQL\nfrom datasette.resources import TableResource\n\n\n@pytest.mark.asyncio\nasync def test_multiple_restriction_sources_intersect():\n    \"\"\"\n    Test that when multiple plugins return restriction_sql, they are INTERSECTed.\n\n    This tests the case where both actor _r restrictions AND a plugin\n    provide restriction_sql - both must pass for access to be granted.\n    \"\"\"\n    from datasette import hookimpl\n\n    class RestrictivePlugin:\n        __name__ = \"RestrictivePlugin\"\n\n        @hookimpl\n        def permission_resources_sql(self, datasette, actor, action):\n            # Plugin adds additional restriction: only db1_multi_intersect allowed\n            if action == \"view-table\":\n                return PermissionSQL(\n                    restriction_sql=\"SELECT 'db1_multi_intersect' AS parent, NULL AS child\",\n                    params={},\n                )\n            return None\n\n    plugin = RestrictivePlugin()\n\n    ds = Datasette()\n    await ds.invoke_startup()\n    ds.pm.register(plugin, name=\"restrictive_plugin\")\n\n    try:\n        db1 = ds.add_memory_database(\"db1_multi_intersect\")\n        db2 = ds.add_memory_database(\"db2_multi_intersect\")\n        await db1.execute_write(\"CREATE TABLE t1 (id INTEGER)\")\n        await db2.execute_write(\"CREATE TABLE t1 (id INTEGER)\")\n        await ds._refresh_schemas()  # Populate catalog tables\n\n        # Actor has restrictions allowing both databases\n        # But plugin only allows db1_multi_intersect\n        # INTERSECT means only db1_multi_intersect/t1 should pass\n        actor = {\n            \"id\": \"user\",\n            \"_r\": {\"d\": {\"db1_multi_intersect\": [\"vt\"], \"db2_multi_intersect\": [\"vt\"]}},\n        }\n\n        page = await ds.allowed_resources(\"view-table\", actor)\n        resources = {(r.parent, r.child) for r in page.resources}\n\n        # Should only see db1_multi_intersect/t1 (intersection of actor restrictions and plugin restrictions)\n        assert (\"db1_multi_intersect\", \"t1\") in resources\n        assert (\"db2_multi_intersect\", \"t1\") not in resources\n    finally:\n        ds.pm.unregister(name=\"restrictive_plugin\")\n\n\n@pytest.mark.asyncio\nasync def test_restriction_sql_with_overlapping_databases_and_tables():\n    \"\"\"\n    Test actor with both database-level and table-level restrictions for same database.\n\n    When actor has:\n    - Database-level: db1_overlapping allowed (all tables)\n    - Table-level: db1_overlapping/t1 allowed\n\n    Both entries are UNION'd (OR'ed) within the actor's restrictions.\n    Database-level restriction allows ALL tables, so table-level is redundant.\n    \"\"\"\n    ds = Datasette()\n    await ds.invoke_startup()\n    db = ds.add_memory_database(\"db1_overlapping\")\n    await db.execute_write(\"CREATE TABLE t1 (id INTEGER)\")\n    await db.execute_write(\"CREATE TABLE t2 (id INTEGER)\")\n    await ds._refresh_schemas()\n\n    # Actor has BOTH database-level (db1_overlapping all tables) AND table-level (db1_overlapping/t1 only)\n    actor = {\n        \"id\": \"user\",\n        \"_r\": {\n            \"d\": {\n                \"db1_overlapping\": [\"vt\"]\n            },  # Database-level: all tables in db1_overlapping\n            \"r\": {\n                \"db1_overlapping\": {\"t1\": [\"vt\"]}\n            },  # Table-level: only t1 in db1_overlapping\n        },\n    }\n\n    # Within actor restrictions, entries are UNION'd (OR'ed):\n    # - Database level allows: (db1_overlapping, NULL) → matches all tables via hierarchical matching\n    # - Table level allows: (db1_overlapping, t1) → redundant, already covered by database level\n    # Result: Both tables are allowed\n    page = await ds.allowed_resources(\"view-table\", actor)\n    resources = {(r.parent, r.child) for r in page.resources}\n\n    assert (\"db1_overlapping\", \"t1\") in resources\n    # Database-level restriction allows all tables, so t2 is also allowed\n    assert (\"db1_overlapping\", \"t2\") in resources\n\n\n@pytest.mark.asyncio\nasync def test_restriction_sql_empty_allowlist_query():\n    \"\"\"\n    Test the specific SQL query generated when action is not in allowlist.\n\n    actor_restrictions_sql() returns \"SELECT NULL AS parent, NULL AS child WHERE 0\"\n    Verify this produces an empty result set.\n    \"\"\"\n    ds = Datasette()\n    await ds.invoke_startup()\n    db = ds.add_memory_database(\"db1_empty_allowlist\")\n    await db.execute_write(\"CREATE TABLE t1 (id INTEGER)\")\n    await ds._refresh_schemas()\n\n    # Actor has restrictions but action not in allowlist\n    actor = {\"id\": \"user\", \"_r\": {\"r\": {\"db1_empty_allowlist\": {\"t1\": [\"vt\"]}}}}\n\n    # Try to view-database (only view-table is in allowlist)\n    page = await ds.allowed_resources(\"view-database\", actor)\n\n    # Should be empty\n    assert len(page.resources) == 0\n\n\n@pytest.mark.asyncio\nasync def test_restriction_sql_with_pagination():\n    \"\"\"\n    Test that restrictions work correctly with keyset pagination.\n    \"\"\"\n    ds = Datasette()\n    await ds.invoke_startup()\n    db = ds.add_memory_database(\"db1_pagination\")\n\n    # Create many tables\n    for i in range(10):\n        await db.execute_write(f\"CREATE TABLE t{i:02d} (id INTEGER)\")\n    await ds._refresh_schemas()\n\n    # Actor restricted to only odd-numbered tables\n    restrictions = {\"r\": {\"db1_pagination\": {}}}\n    for i in range(10):\n        if i % 2 == 1:  # Only odd tables\n            restrictions[\"r\"][\"db1_pagination\"][f\"t{i:02d}\"] = [\"vt\"]\n\n    actor = {\"id\": \"user\", \"_r\": restrictions}\n\n    # Get first page with small limit\n    page1 = await ds.allowed_resources(\n        \"view-table\", actor, parent=\"db1_pagination\", limit=2\n    )\n    assert len(page1.resources) == 2\n    assert page1.next is not None\n\n    # Get second page using next token\n    page2 = await ds.allowed_resources(\n        \"view-table\", actor, parent=\"db1_pagination\", limit=2, next=page1.next\n    )\n    assert len(page2.resources) == 2\n\n    # Should have no overlap\n    page1_ids = {r.child for r in page1.resources}\n    page2_ids = {r.child for r in page2.resources}\n    assert page1_ids.isdisjoint(page2_ids)\n\n    # All should be odd-numbered tables\n    all_ids = page1_ids | page2_ids\n    for table_id in all_ids:\n        table_num = int(table_id[1:])  # Extract number from \"t01\", \"t03\", etc.\n        assert table_num % 2 == 1, f\"Table {table_id} should be odd-numbered\"\n\n\n@pytest.mark.asyncio\nasync def test_also_requires_with_restrictions():\n    \"\"\"\n    Test that also_requires actions properly respect restrictions.\n\n    execute-sql requires view-database. With restrictions, both must pass.\n    \"\"\"\n    ds = Datasette()\n    await ds.invoke_startup()\n    ds.add_memory_database(\"db1_also_requires\")\n    ds.add_memory_database(\"db2_also_requires\")\n    await ds._refresh_schemas()\n\n    # Actor restricted to only db1_also_requires for view-database\n    # execute-sql requires view-database, so should only work on db1_also_requires\n    actor = {\n        \"id\": \"user\",\n        \"_r\": {\n            \"d\": {\n                \"db1_also_requires\": [\"vd\", \"es\"],\n                \"db2_also_requires\": [\n                    \"es\"\n                ],  # They have execute-sql but not view-database\n            }\n        },\n    }\n\n    # db1_also_requires should allow execute-sql\n    result = await ds.allowed(\n        action=\"execute-sql\",\n        resource=TableResource(\"db1_also_requires\", None),\n        actor=actor,\n    )\n    assert result is True\n\n    # db2_also_requires should not (they have execute-sql but not view-database)\n    result = await ds.allowed(\n        action=\"execute-sql\",\n        resource=TableResource(\"db2_also_requires\", None),\n        actor=actor,\n    )\n    assert result is False\n\n\n@pytest.mark.asyncio\nasync def test_restriction_abbreviations_and_full_names():\n    \"\"\"\n    Test that both abbreviations and full action names work in restrictions.\n    \"\"\"\n    ds = Datasette()\n    await ds.invoke_startup()\n    db = ds.add_memory_database(\"db1_abbrev\")\n    await db.execute_write(\"CREATE TABLE t1 (id INTEGER)\")\n    await ds._refresh_schemas()\n\n    # Test with abbreviation\n    actor_abbr = {\"id\": \"user\", \"_r\": {\"r\": {\"db1_abbrev\": {\"t1\": [\"vt\"]}}}}\n    result = await ds.allowed(\n        action=\"view-table\",\n        resource=TableResource(\"db1_abbrev\", \"t1\"),\n        actor=actor_abbr,\n    )\n    assert result is True\n\n    # Test with full name\n    actor_full = {\"id\": \"user\", \"_r\": {\"r\": {\"db1_abbrev\": {\"t1\": [\"view-table\"]}}}}\n    result = await ds.allowed(\n        action=\"view-table\",\n        resource=TableResource(\"db1_abbrev\", \"t1\"),\n        actor=actor_full,\n    )\n    assert result is True\n\n    # Test with mixed\n    actor_mixed = {\"id\": \"user\", \"_r\": {\"d\": {\"db1_abbrev\": [\"view-database\", \"vt\"]}}}\n    result = await ds.allowed(\n        action=\"view-table\",\n        resource=TableResource(\"db1_abbrev\", \"t1\"),\n        actor=actor_mixed,\n    )\n    assert result is True\n\n\n@pytest.mark.asyncio\nasync def test_permission_resources_sql_multiple_restriction_sources_intersect():\n    \"\"\"\n    Test that when multiple plugins return restriction_sql, they are INTERSECTed.\n\n    This tests the case where both actor _r restrictions AND a plugin\n    provide restriction_sql - both must pass for access to be granted.\n    \"\"\"\n    from datasette import hookimpl\n\n    class RestrictivePlugin:\n        __name__ = \"RestrictivePlugin\"\n\n        @hookimpl\n        def permission_resources_sql(self, datasette, actor, action):\n            # Plugin adds additional restriction: only db1_multi_restrictions allowed\n            if action == \"view-table\":\n                return PermissionSQL(\n                    restriction_sql=\"SELECT 'db1_multi_restrictions' AS parent, NULL AS child\",\n                    params={},\n                )\n            return None\n\n    plugin = RestrictivePlugin()\n\n    ds = Datasette()\n    await ds.invoke_startup()\n    ds.pm.register(plugin, name=\"restrictive_plugin\")\n\n    try:\n        db1 = ds.add_memory_database(\"db1_multi_restrictions\")\n        db2 = ds.add_memory_database(\"db2_multi_restrictions\")\n        await db1.execute_write(\"CREATE TABLE t1 (id INTEGER)\")\n        await db2.execute_write(\"CREATE TABLE t1 (id INTEGER)\")\n        await ds._refresh_schemas()  # Populate catalog tables\n\n        # Actor has restrictions allowing both databases\n        # But plugin only allows db1\n        # INTERSECT means only db1/t1 should pass\n        actor = {\n            \"id\": \"user\",\n            \"_r\": {\n                \"d\": {\n                    \"db1_multi_restrictions\": [\"vt\"],\n                    \"db2_multi_restrictions\": [\"vt\"],\n                }\n            },\n        }\n\n        page = await ds.allowed_resources(\"view-table\", actor)\n        resources = {(r.parent, r.child) for r in page.resources}\n\n        # Should only see db1/t1 (intersection of actor restrictions and plugin restrictions)\n        assert (\"db1_multi_restrictions\", \"t1\") in resources\n        assert (\"db2_multi_restrictions\", \"t1\") not in resources\n    finally:\n        ds.pm.unregister(name=\"restrictive_plugin\")\n"
  },
  {
    "path": "tests/test_routes.py",
    "content": "from datasette.app import Datasette, Database\nfrom datasette.utils import resolve_routes\nimport pytest\nimport pytest_asyncio\n\n\n@pytest.fixture(scope=\"session\")\ndef routes():\n    ds = Datasette()\n    return ds._routes()\n\n\n@pytest.mark.parametrize(\n    \"path,expected_name,expected_matches\",\n    (\n        (\"/\", \"IndexView\", {\"format\": None}),\n        (\"/foo\", \"DatabaseView\", {\"format\": None, \"database\": \"foo\"}),\n        (\"/foo.csv\", \"DatabaseView\", {\"format\": \"csv\", \"database\": \"foo\"}),\n        (\"/foo.json\", \"DatabaseView\", {\"format\": \"json\", \"database\": \"foo\"}),\n        (\"/foo.humbug\", \"DatabaseView\", {\"format\": \"humbug\", \"database\": \"foo\"}),\n        (\n            \"/foo/humbug\",\n            \"table_view\",\n            {\"database\": \"foo\", \"table\": \"humbug\", \"format\": None},\n        ),\n        (\n            \"/foo/humbug.json\",\n            \"table_view\",\n            {\"database\": \"foo\", \"table\": \"humbug\", \"format\": \"json\"},\n        ),\n        (\n            \"/foo/humbug.blah\",\n            \"table_view\",\n            {\"database\": \"foo\", \"table\": \"humbug\", \"format\": \"blah\"},\n        ),\n        (\n            \"/foo/humbug/1\",\n            \"RowView\",\n            {\"format\": None, \"database\": \"foo\", \"pks\": \"1\", \"table\": \"humbug\"},\n        ),\n        (\n            \"/foo/humbug/1.json\",\n            \"RowView\",\n            {\"format\": \"json\", \"database\": \"foo\", \"pks\": \"1\", \"table\": \"humbug\"},\n        ),\n    ),\n)\ndef test_routes(routes, path, expected_name, expected_matches):\n    match, view = resolve_routes(routes, path)\n    if expected_name is None:\n        assert match is None\n    else:\n        assert (\n            view.__name__ == expected_name or view.view_class.__name__ == expected_name\n        )\n        assert match.groupdict() == expected_matches\n\n\n@pytest_asyncio.fixture\nasync def ds_with_route():\n    ds = Datasette()\n    await ds.invoke_startup()\n    ds.remove_database(\"_memory\")\n    db = Database(ds, is_memory=True, memory_name=\"route-name-db\")\n    ds.add_database(db, name=\"original-name\", route=\"custom-route-name\")\n    await db.execute_write_script(\"\"\"\n        create table if not exists t (id integer primary key);\n        insert or replace into t (id) values (1);\n    \"\"\")\n    return ds\n\n\n@pytest.mark.asyncio\nasync def test_db_with_route_databases(ds_with_route):\n    response = await ds_with_route.client.get(\"/-/databases.json\")\n    assert response.json()[0] == {\n        \"name\": \"original-name\",\n        \"route\": \"custom-route-name\",\n        \"path\": None,\n        \"size\": 0,\n        \"is_mutable\": True,\n        \"is_memory\": True,\n        \"hash\": None,\n    }\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"path,expected_status\",\n    (\n        (\"/\", 200),\n        (\"/original-name\", 404),\n        (\"/original-name/t\", 404),\n        (\"/original-name/t/1\", 404),\n        (\"/custom-route-name\", 200),\n        (\"/custom-route-name/-/query?sql=select+id+from+t\", 200),\n        (\"/custom-route-name/t\", 200),\n        (\"/custom-route-name/t/1\", 200),\n    ),\n)\nasync def test_db_with_route_that_does_not_match_name(\n    ds_with_route, path, expected_status\n):\n    response = await ds_with_route.client.get(path)\n    assert response.status_code == expected_status\n    # There should be links to custom-route-name but none to original-name\n    if response.status_code == 200:\n        assert \"/custom-route-name\" in response.text\n        assert \"/original-name\" not in response.text\n"
  },
  {
    "path": "tests/test_schema_endpoints.py",
    "content": "import pytest\nimport pytest_asyncio\nfrom datasette.app import Datasette\n\n\n@pytest_asyncio.fixture(scope=\"module\")\nasync def schema_ds():\n    \"\"\"Create a Datasette instance with test databases and permission config.\"\"\"\n    ds = Datasette(\n        config={\n            \"databases\": {\n                \"schema_private_db\": {\"allow\": {\"id\": \"root\"}},\n            }\n        }\n    )\n\n    # Create public database with multiple tables\n    public_db = ds.add_memory_database(\"schema_public_db\")\n    await public_db.execute_write(\n        \"CREATE TABLE IF NOT EXISTS users (id INTEGER PRIMARY KEY, name TEXT)\"\n    )\n    await public_db.execute_write(\n        \"CREATE TABLE IF NOT EXISTS posts (id INTEGER PRIMARY KEY, title TEXT)\"\n    )\n    await public_db.execute_write(\n        \"CREATE VIEW IF NOT EXISTS recent_posts AS SELECT * FROM posts ORDER BY id DESC\"\n    )\n\n    # Create a database with restricted access (requires root permission)\n    private_db = ds.add_memory_database(\"schema_private_db\")\n    await private_db.execute_write(\n        \"CREATE TABLE IF NOT EXISTS secret_data (id INTEGER PRIMARY KEY, value TEXT)\"\n    )\n\n    # Create an empty database\n    ds.add_memory_database(\"schema_empty_db\")\n\n    return ds\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"format_ext,expected_in_content\",\n    [\n        (\"json\", None),\n        (\"md\", [\"# Schema for\", \"```sql\"]),\n        (\"\", [\"Schema for\", \"CREATE TABLE\"]),\n    ],\n)\nasync def test_database_schema_formats(schema_ds, format_ext, expected_in_content):\n    \"\"\"Test /database/-/schema endpoint in different formats.\"\"\"\n    url = \"/schema_public_db/-/schema\"\n    if format_ext:\n        url += f\".{format_ext}\"\n    response = await schema_ds.client.get(url)\n    assert response.status_code == 200\n\n    if format_ext == \"json\":\n        data = response.json()\n        assert \"database\" in data\n        assert data[\"database\"] == \"schema_public_db\"\n        assert \"schema\" in data\n        assert \"CREATE TABLE users\" in data[\"schema\"]\n    else:\n        content = response.text\n        for expected in expected_in_content:\n            assert expected in content\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"format_ext,expected_in_content\",\n    [\n        (\"json\", None),\n        (\"md\", [\"# Schema for\", \"```sql\"]),\n        (\"\", [\"Schema for all databases\"]),\n    ],\n)\nasync def test_instance_schema_formats(schema_ds, format_ext, expected_in_content):\n    \"\"\"Test /-/schema endpoint in different formats.\"\"\"\n    url = \"/-/schema\"\n    if format_ext:\n        url += f\".{format_ext}\"\n    response = await schema_ds.client.get(url)\n    assert response.status_code == 200\n\n    if format_ext == \"json\":\n        data = response.json()\n        assert \"schemas\" in data\n        assert isinstance(data[\"schemas\"], list)\n        db_names = [item[\"database\"] for item in data[\"schemas\"]]\n        # Should see schema_public_db and schema_empty_db, but not schema_private_db (anonymous user)\n        assert \"schema_public_db\" in db_names\n        assert \"schema_empty_db\" in db_names\n        assert \"schema_private_db\" not in db_names\n        # Check schemas are present\n        for item in data[\"schemas\"]:\n            if item[\"database\"] == \"schema_public_db\":\n                assert \"CREATE TABLE users\" in item[\"schema\"]\n    else:\n        content = response.text\n        for expected in expected_in_content:\n            assert expected in content\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"format_ext,expected_in_content\",\n    [\n        (\"json\", None),\n        (\"md\", [\"# Schema for\", \"```sql\"]),\n        (\"\", [\"Schema for users\"]),\n    ],\n)\nasync def test_table_schema_formats(schema_ds, format_ext, expected_in_content):\n    \"\"\"Test /database/table/-/schema endpoint in different formats.\"\"\"\n    url = \"/schema_public_db/users/-/schema\"\n    if format_ext:\n        url += f\".{format_ext}\"\n    response = await schema_ds.client.get(url)\n    assert response.status_code == 200\n\n    if format_ext == \"json\":\n        data = response.json()\n        assert \"database\" in data\n        assert data[\"database\"] == \"schema_public_db\"\n        assert \"table\" in data\n        assert data[\"table\"] == \"users\"\n        assert \"schema\" in data\n        assert \"CREATE TABLE users\" in data[\"schema\"]\n    else:\n        content = response.text\n        for expected in expected_in_content:\n            assert expected in content\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"url\",\n    [\n        \"/schema_private_db/-/schema.json\",\n        \"/schema_private_db/secret_data/-/schema.json\",\n    ],\n)\nasync def test_schema_permission_enforcement(schema_ds, url):\n    \"\"\"Test that permissions are enforced for schema endpoints.\"\"\"\n    # Anonymous user should get 403\n    response = await schema_ds.client.get(url)\n    assert response.status_code == 403\n\n    # Authenticated user with permission should succeed\n    response = await schema_ds.client.get(\n        url,\n        cookies={\"ds_actor\": schema_ds.client.actor_cookie({\"id\": \"root\"})},\n    )\n    assert response.status_code == 200\n\n\n@pytest.mark.asyncio\nasync def test_instance_schema_respects_database_permissions(schema_ds):\n    \"\"\"Test that /-/schema only shows databases the user can view.\"\"\"\n    # Anonymous user should only see public databases\n    response = await schema_ds.client.get(\"/-/schema.json\")\n    assert response.status_code == 200\n    data = response.json()\n    db_names = [item[\"database\"] for item in data[\"schemas\"]]\n    assert \"schema_public_db\" in db_names\n    assert \"schema_empty_db\" in db_names\n    assert \"schema_private_db\" not in db_names\n\n    # Authenticated user should see all databases\n    response = await schema_ds.client.get(\n        \"/-/schema.json\",\n        cookies={\"ds_actor\": schema_ds.client.actor_cookie({\"id\": \"root\"})},\n    )\n    assert response.status_code == 200\n    data = response.json()\n    db_names = [item[\"database\"] for item in data[\"schemas\"]]\n    assert \"schema_public_db\" in db_names\n    assert \"schema_empty_db\" in db_names\n    assert \"schema_private_db\" in db_names\n\n\n@pytest.mark.asyncio\nasync def test_database_schema_with_multiple_tables(schema_ds):\n    \"\"\"Test schema with multiple tables in a database.\"\"\"\n    response = await schema_ds.client.get(\"/schema_public_db/-/schema.json\")\n    assert response.status_code == 200\n    data = response.json()\n    schema = data[\"schema\"]\n\n    # All objects should be in the schema\n    assert \"CREATE TABLE users\" in schema\n    assert \"CREATE TABLE posts\" in schema\n    assert \"CREATE VIEW recent_posts\" in schema\n\n\n@pytest.mark.asyncio\nasync def test_empty_database_schema(schema_ds):\n    \"\"\"Test schema for an empty database.\"\"\"\n    response = await schema_ds.client.get(\"/schema_empty_db/-/schema.json\")\n    assert response.status_code == 200\n    data = response.json()\n    assert data[\"database\"] == \"schema_empty_db\"\n    assert data[\"schema\"] == \"\"\n\n\n@pytest.mark.asyncio\nasync def test_database_not_exists(schema_ds):\n    \"\"\"Test schema for a non-existent database returns 404.\"\"\"\n    # Test JSON format\n    response = await schema_ds.client.get(\"/nonexistent_db/-/schema.json\")\n    assert response.status_code == 404\n    data = response.json()\n    assert data[\"ok\"] is False\n    assert \"not found\" in data[\"error\"].lower()\n\n    # Test HTML format (returns text)\n    response = await schema_ds.client.get(\"/nonexistent_db/-/schema\")\n    assert response.status_code == 404\n    assert \"not found\" in response.text.lower()\n\n    # Test Markdown format (returns text)\n    response = await schema_ds.client.get(\"/nonexistent_db/-/schema.md\")\n    assert response.status_code == 404\n    assert \"not found\" in response.text.lower()\n\n\n@pytest.mark.asyncio\nasync def test_table_not_exists(schema_ds):\n    \"\"\"Test schema for a non-existent table returns 404.\"\"\"\n    # Test JSON format\n    response = await schema_ds.client.get(\"/schema_public_db/nonexistent/-/schema.json\")\n    assert response.status_code == 404\n    data = response.json()\n    assert data[\"ok\"] is False\n    assert \"not found\" in data[\"error\"].lower()\n\n    # Test HTML format (returns text)\n    response = await schema_ds.client.get(\"/schema_public_db/nonexistent/-/schema\")\n    assert response.status_code == 404\n    assert \"not found\" in response.text.lower()\n\n    # Test Markdown format (returns text)\n    response = await schema_ds.client.get(\"/schema_public_db/nonexistent/-/schema.md\")\n    assert response.status_code == 404\n    assert \"not found\" in response.text.lower()\n"
  },
  {
    "path": "tests/test_search_tables.py",
    "content": "\"\"\"\nTests for special endpoints in datasette/views/special.py\n\"\"\"\n\nimport pytest\nimport pytest_asyncio\nfrom datasette.app import Datasette\n\n\n@pytest_asyncio.fixture\nasync def ds_with_tables():\n    \"\"\"Create a Datasette instance with some tables for searching.\"\"\"\n    ds = Datasette(\n        config={\n            \"databases\": {\n                \"content\": {\n                    \"allow\": {\"id\": \"*\"},  # Allow all authenticated users\n                    \"tables\": {\n                        \"articles\": {\n                            \"allow\": {\"id\": \"editor\"},  # Only editor can view\n                        },\n                        \"comments\": {\n                            \"allow\": True,  # Everyone can view\n                        },\n                    },\n                },\n                \"private\": {\n                    \"allow\": False,  # Deny everyone\n                },\n            }\n        }\n    )\n    await ds.invoke_startup()\n\n    # Add content database with some tables\n    content_db = ds.add_memory_database(\"content\")\n    await content_db.execute_write(\n        \"CREATE TABLE IF NOT EXISTS articles (id INTEGER PRIMARY KEY, title TEXT)\"\n    )\n    await content_db.execute_write(\n        \"CREATE TABLE IF NOT EXISTS comments (id INTEGER PRIMARY KEY, body TEXT)\"\n    )\n    await content_db.execute_write(\n        \"CREATE TABLE IF NOT EXISTS users (id INTEGER PRIMARY KEY, name TEXT)\"\n    )\n\n    # Add private database with a table\n    private_db = ds.add_memory_database(\"private\")\n    await private_db.execute_write(\n        \"CREATE TABLE IF NOT EXISTS secrets (id INTEGER PRIMARY KEY, data TEXT)\"\n    )\n\n    # Add another public database\n    public_db = ds.add_memory_database(\"public\")\n    await public_db.execute_write(\n        \"CREATE TABLE IF NOT EXISTS articles (id INTEGER PRIMARY KEY, content TEXT)\"\n    )\n\n    return ds\n\n\n# /-/tables.json tests\n@pytest.mark.asyncio\nasync def test_tables_basic_search(ds_with_tables):\n    \"\"\"Test basic table search functionality.\"\"\"\n    # Search for \"articles\" - should find it in both content and public databases\n    # but only return public.articles for anonymous user (content.articles requires auth)\n    response = await ds_with_tables.client.get(\"/-/tables.json?q=articles\")\n    assert response.status_code == 200\n    data = response.json()\n\n    # Should only see public.articles (content.articles restricted to authenticated users)\n    assert \"matches\" in data\n    assert len(data[\"matches\"]) == 1\n\n    match = data[\"matches\"][0]\n    assert \"url\" in match\n    assert \"name\" in match\n    assert match[\"name\"] == \"public: articles\"\n    assert \"/public/articles\" in match[\"url\"]\n\n\n@pytest.mark.asyncio\nasync def test_tables_search_with_auth(ds_with_tables):\n    \"\"\"Test that authenticated users see more tables.\"\"\"\n    # Editor user should see content.articles\n    response = await ds_with_tables.client.get(\n        \"/-/tables.json?q=articles\",\n        cookies={\"ds_actor\": ds_with_tables.client.actor_cookie({\"id\": \"editor\"})},\n    )\n    assert response.status_code == 200\n    data = response.json()\n\n    # Should see both content.articles and public.articles\n    assert len(data[\"matches\"]) == 2\n\n    names = {match[\"name\"] for match in data[\"matches\"]}\n    assert names == {\"content: articles\", \"public: articles\"}\n\n\n@pytest.mark.asyncio\nasync def test_tables_search_partial_match(ds_with_tables):\n    \"\"\"Test that search matches partial table names.\"\"\"\n    # Search for \"com\" should match \"comments\"\n    response = await ds_with_tables.client.get(\n        \"/-/tables.json?q=com\",\n        cookies={\"ds_actor\": ds_with_tables.client.actor_cookie({\"id\": \"user\"})},\n    )\n    assert response.status_code == 200\n    data = response.json()\n\n    assert len(data[\"matches\"]) == 1\n    assert data[\"matches\"][0][\"name\"] == \"content: comments\"\n\n\n@pytest.mark.asyncio\nasync def test_tables_search_respects_database_permissions(ds_with_tables):\n    \"\"\"Test that tables from denied databases are not shown.\"\"\"\n    # Search for \"secrets\" which is in the private database\n    # Even authenticated users shouldn't see it because database is denied\n    response = await ds_with_tables.client.get(\n        \"/-/tables.json?q=secrets\",\n        cookies={\"ds_actor\": ds_with_tables.client.actor_cookie({\"id\": \"user\"})},\n    )\n    assert response.status_code == 200\n    data = response.json()\n\n    # Should not see secrets table from private database\n    assert len(data[\"matches\"]) == 0\n\n\n@pytest.mark.asyncio\nasync def test_tables_search_respects_table_permissions(ds_with_tables):\n    \"\"\"Test that tables with specific permissions are filtered correctly.\"\"\"\n    # Regular authenticated user searching for \"users\"\n    response = await ds_with_tables.client.get(\n        \"/-/tables.json?q=users\",\n        cookies={\"ds_actor\": ds_with_tables.client.actor_cookie({\"id\": \"regular\"})},\n    )\n    assert response.status_code == 200\n    data = response.json()\n\n    # Should see content.users (authenticated users can view content database)\n    assert len(data[\"matches\"]) == 1\n    assert data[\"matches\"][0][\"name\"] == \"content: users\"\n\n\n@pytest.mark.asyncio\nasync def test_tables_search_response_structure(ds_with_tables):\n    \"\"\"Test that response has correct structure.\"\"\"\n    response = await ds_with_tables.client.get(\n        \"/-/tables.json?q=users\",\n        cookies={\"ds_actor\": ds_with_tables.client.actor_cookie({\"id\": \"user\"})},\n    )\n    assert response.status_code == 200\n    data = response.json()\n\n    assert \"matches\" in data\n    assert isinstance(data[\"matches\"], list)\n\n    if data[\"matches\"]:\n        match = data[\"matches\"][0]\n        assert \"url\" in match\n        assert \"name\" in match\n        assert isinstance(match[\"url\"], str)\n        assert isinstance(match[\"name\"], str)\n        # Name should be in format \"database: table\"\n        assert \": \" in match[\"name\"]\n"
  },
  {
    "path": "tests/test_spatialite.py",
    "content": "from datasette.app import Datasette\nfrom datasette.utils import find_spatialite, SpatialiteNotFound, SPATIALITE_FUNCTIONS\nfrom .utils import has_load_extension\nimport pytest\n\n\ndef has_spatialite():\n    try:\n        find_spatialite()\n        return True\n    except SpatialiteNotFound:\n        return False\n\n\n@pytest.mark.asyncio\n@pytest.mark.skipif(not has_spatialite(), reason=\"Requires SpatiaLite\")\n@pytest.mark.skipif(not has_load_extension(), reason=\"Requires enable_load_extension\")\nasync def test_spatialite_version_info():\n    ds = Datasette(sqlite_extensions=[\"spatialite\"])\n    response = await ds.client.get(\"/-/versions.json\")\n    assert response.status_code == 200\n    spatialite = response.json()[\"sqlite\"][\"extensions\"][\"spatialite\"]\n    assert set(SPATIALITE_FUNCTIONS) == set(spatialite)\n"
  },
  {
    "path": "tests/test_table_api.py",
    "content": "from datasette.utils import detect_json1\nfrom datasette.utils.sqlite import sqlite_version\nfrom .fixtures import generate_compound_rows, generate_sortable_rows, make_app_client\nimport json\nimport pytest\nimport urllib\n\n\n@pytest.mark.asyncio\nasync def test_table_json(ds_client):\n    response = await ds_client.get(\"/fixtures/simple_primary_key.json?_extra=query\")\n    assert response.status_code == 200\n    data = response.json()\n    assert (\n        data[\"query\"][\"sql\"]\n        == \"select id, content from simple_primary_key order by id limit 51\"\n    )\n    assert data[\"query\"][\"params\"] == {}\n    assert data[\"rows\"] == [\n        {\"id\": 1, \"content\": \"hello\"},\n        {\"id\": 2, \"content\": \"world\"},\n        {\"id\": 3, \"content\": \"\"},\n        {\"id\": 4, \"content\": \"RENDER_CELL_DEMO\"},\n        {\"id\": 5, \"content\": \"RENDER_CELL_ASYNC\"},\n    ]\n\n\n@pytest.mark.asyncio\nasync def test_table_not_exists_json(ds_client):\n    assert (await ds_client.get(\"/fixtures/blah.json\")).json() == {\n        \"ok\": False,\n        \"error\": \"Table not found\",\n        \"status\": 404,\n        \"title\": None,\n    }\n\n\n@pytest.mark.asyncio\nasync def test_table_shape_arrays(ds_client):\n    response = await ds_client.get(\"/fixtures/simple_primary_key.json?_shape=arrays\")\n    assert response.json()[\"rows\"] == [\n        [1, \"hello\"],\n        [2, \"world\"],\n        [3, \"\"],\n        [4, \"RENDER_CELL_DEMO\"],\n        [5, \"RENDER_CELL_ASYNC\"],\n    ]\n\n\n@pytest.mark.asyncio\nasync def test_table_shape_arrayfirst(ds_client):\n    response = await ds_client.get(\n        \"/fixtures/-/query.json?\"\n        + urllib.parse.urlencode(\n            {\n                \"sql\": \"select content from simple_primary_key order by id\",\n                \"_shape\": \"arrayfirst\",\n            }\n        )\n    )\n    assert response.json() == [\n        \"hello\",\n        \"world\",\n        \"\",\n        \"RENDER_CELL_DEMO\",\n        \"RENDER_CELL_ASYNC\",\n    ]\n\n\n@pytest.mark.asyncio\nasync def test_table_shape_objects(ds_client):\n    response = await ds_client.get(\"/fixtures/simple_primary_key.json?_shape=objects\")\n    assert response.json()[\"rows\"] == [\n        {\"id\": 1, \"content\": \"hello\"},\n        {\"id\": 2, \"content\": \"world\"},\n        {\"id\": 3, \"content\": \"\"},\n        {\"id\": 4, \"content\": \"RENDER_CELL_DEMO\"},\n        {\"id\": 5, \"content\": \"RENDER_CELL_ASYNC\"},\n    ]\n\n\n@pytest.mark.asyncio\nasync def test_table_shape_array(ds_client):\n    response = await ds_client.get(\"/fixtures/simple_primary_key.json?_shape=array\")\n    assert response.json() == [\n        {\"id\": 1, \"content\": \"hello\"},\n        {\"id\": 2, \"content\": \"world\"},\n        {\"id\": 3, \"content\": \"\"},\n        {\"id\": 4, \"content\": \"RENDER_CELL_DEMO\"},\n        {\"id\": 5, \"content\": \"RENDER_CELL_ASYNC\"},\n    ]\n\n\n@pytest.mark.asyncio\nasync def test_table_shape_array_nl(ds_client):\n    response = await ds_client.get(\n        \"/fixtures/simple_primary_key.json?_shape=array&_nl=on\"\n    )\n    lines = response.text.split(\"\\n\")\n    results = [json.loads(line) for line in lines]\n    assert [\n        {\"id\": 1, \"content\": \"hello\"},\n        {\"id\": 2, \"content\": \"world\"},\n        {\"id\": 3, \"content\": \"\"},\n        {\"id\": 4, \"content\": \"RENDER_CELL_DEMO\"},\n        {\"id\": 5, \"content\": \"RENDER_CELL_ASYNC\"},\n    ] == results\n\n\n@pytest.mark.asyncio\nasync def test_table_shape_invalid(ds_client):\n    response = await ds_client.get(\"/fixtures/simple_primary_key.json?_shape=invalid\")\n    assert response.json() == {\n        \"ok\": False,\n        \"error\": \"Invalid _shape: invalid\",\n        \"status\": 400,\n        \"title\": None,\n    }\n\n\n@pytest.mark.asyncio\nasync def test_table_shape_object(ds_client):\n    response = await ds_client.get(\"/fixtures/simple_primary_key.json?_shape=object\")\n    assert response.json() == {\n        \"1\": {\"id\": 1, \"content\": \"hello\"},\n        \"2\": {\"id\": 2, \"content\": \"world\"},\n        \"3\": {\"id\": 3, \"content\": \"\"},\n        \"4\": {\"id\": 4, \"content\": \"RENDER_CELL_DEMO\"},\n        \"5\": {\"id\": 5, \"content\": \"RENDER_CELL_ASYNC\"},\n    }\n\n\n@pytest.mark.asyncio\nasync def test_table_shape_object_compound_primary_key(ds_client):\n    response = await ds_client.get(\"/fixtures/compound_primary_key.json?_shape=object\")\n    assert response.json() == {\n        \"a,b\": {\"pk1\": \"a\", \"pk2\": \"b\", \"content\": \"c\"},\n        \"a~2Fb,~2Ec-d\": {\"pk1\": \"a/b\", \"pk2\": \".c-d\", \"content\": \"c\"},\n        \"d,e\": {\"pk1\": \"d\", \"pk2\": \"e\", \"content\": \"RENDER_CELL_DEMO\"},\n    }\n\n\n@pytest.mark.asyncio\nasync def test_table_with_slashes_in_name(ds_client):\n    response = await ds_client.get(\n        \"/fixtures/table~2Fwith~2Fslashes~2Ecsv.json?_shape=objects\"\n    )\n    assert response.status_code == 200\n    data = response.json()\n    assert data[\"rows\"] == [{\"pk\": \"3\", \"content\": \"hey\"}]\n\n\n@pytest.mark.asyncio\nasync def test_table_with_reserved_word_name(ds_client):\n    response = await ds_client.get(\"/fixtures/select.json?_shape=objects\")\n    assert response.status_code == 200\n    data = response.json()\n    assert data[\"rows\"] == [\n        {\n            \"rowid\": 1,\n            \"group\": \"group\",\n            \"having\": \"having\",\n            \"and\": \"and\",\n            \"json\": '{\"href\": \"http://example.com/\", \"label\":\"Example\"}',\n        }\n    ]\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"path,expected_rows,expected_pages\",\n    [\n        (\"/fixtures/no_primary_key.json\", 202, 5),\n        (\"/fixtures/paginated_view.json\", 202, 9),\n        (\"/fixtures/no_primary_key.json?_size=25\", 202, 9),\n        (\"/fixtures/paginated_view.json?_size=50\", 202, 5),\n        (\"/fixtures/paginated_view.json?_size=max\", 202, 3),\n        (\"/fixtures/123_starts_with_digits.json\", 0, 1),\n        # Ensure faceting doesn't break pagination:\n        (\"/fixtures/compound_three_primary_keys.json?_facet=pk1\", 1001, 21),\n        # Paginating while sorted by an expanded foreign key should work\n        (\n            \"/fixtures/roadside_attraction_characteristics.json?_size=2&_sort=attraction_id&_labels=on\",\n            5,\n            3,\n        ),\n    ],\n)\nasync def test_paginate_tables_and_views(\n    ds_client, path, expected_rows, expected_pages\n):\n    fetched = []\n    count = 0\n    while path:\n        if \"?\" in path:\n            path += \"&_extra=next_url\"\n        else:\n            path += \"?_extra=next_url\"\n        response = await ds_client.get(path)\n        assert response.status_code == 200\n        count += 1\n        fetched.extend(response.json()[\"rows\"])\n        path = response.json()[\"next_url\"]\n        if path:\n            assert urllib.parse.urlencode({\"_next\": response.json()[\"next\"]}) in path\n            path = path.replace(\"http://localhost\", \"\")\n        assert count < 30, \"Possible infinite loop detected\"\n\n    assert expected_rows == len(fetched)\n    assert expected_pages == count\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"path,expected_error\",\n    [\n        (\"/fixtures/no_primary_key.json?_size=-4\", \"_size must be a positive integer\"),\n        (\"/fixtures/no_primary_key.json?_size=dog\", \"_size must be a positive integer\"),\n        (\"/fixtures/no_primary_key.json?_size=1001\", \"_size must be <= 100\"),\n    ],\n)\nasync def test_validate_page_size(ds_client, path, expected_error):\n    response = await ds_client.get(path)\n    assert expected_error == response.json()[\"error\"]\n    assert response.status_code == 400\n\n\n@pytest.mark.asyncio\nasync def test_page_size_zero(ds_client):\n    \"\"\"For _size=0 we return the counts, empty rows and no continuation token\"\"\"\n    response = await ds_client.get(\n        \"/fixtures/no_primary_key.json?_size=0&_extra=count,next_url\"\n    )\n    assert response.status_code == 200\n    assert [] == response.json()[\"rows\"]\n    assert 202 == response.json()[\"count\"]\n    assert None is response.json()[\"next\"]\n    assert None is response.json()[\"next_url\"]\n\n\n@pytest.mark.asyncio\nasync def test_paginate_compound_keys(ds_client):\n    fetched = []\n    path = \"/fixtures/compound_three_primary_keys.json?_shape=objects&_extra=next_url\"\n    page = 0\n    while path:\n        page += 1\n        response = await ds_client.get(path)\n        fetched.extend(response.json()[\"rows\"])\n        path = response.json()[\"next_url\"]\n        if path:\n            path = path.replace(\"http://localhost\", \"\")\n        assert page < 100\n    assert 1001 == len(fetched)\n    assert 21 == page\n    # Should be correctly ordered\n    contents = [f[\"content\"] for f in fetched]\n    expected = [r[3] for r in generate_compound_rows(1001)]\n    assert expected == contents\n\n\n@pytest.mark.asyncio\nasync def test_paginate_compound_keys_with_extra_filters(ds_client):\n    fetched = []\n    path = \"/fixtures/compound_three_primary_keys.json?content__contains=d&_shape=objects&_extra=next_url\"\n    page = 0\n    while path:\n        page += 1\n        assert page < 100\n        response = await ds_client.get(path)\n        fetched.extend(response.json()[\"rows\"])\n        path = response.json()[\"next_url\"]\n        if path:\n            path = path.replace(\"http://localhost\", \"\")\n    assert 2 == page\n    expected = [r[3] for r in generate_compound_rows(1001) if \"d\" in r[3]]\n    assert expected == [f[\"content\"] for f in fetched]\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"query_string,sort_key,human_description_en\",\n    [\n        (\"_sort=sortable\", lambda row: row[\"sortable\"], \"sorted by sortable\"),\n        (\n            \"_sort_desc=sortable\",\n            lambda row: -row[\"sortable\"],\n            \"sorted by sortable descending\",\n        ),\n        (\n            \"_sort=sortable_with_nulls\",\n            lambda row: (\n                1 if row[\"sortable_with_nulls\"] is not None else 0,\n                row[\"sortable_with_nulls\"],\n            ),\n            \"sorted by sortable_with_nulls\",\n        ),\n        (\n            \"_sort_desc=sortable_with_nulls\",\n            lambda row: (\n                1 if row[\"sortable_with_nulls\"] is None else 0,\n                (\n                    -row[\"sortable_with_nulls\"]\n                    if row[\"sortable_with_nulls\"] is not None\n                    else 0\n                ),\n                row[\"content\"],\n            ),\n            \"sorted by sortable_with_nulls descending\",\n        ),\n        # text column contains '$null' - ensure it doesn't confuse pagination:\n        (\"_sort=text\", lambda row: row[\"text\"], \"sorted by text\"),\n        # Still works if sort column removed using _col=\n        (\"_sort=text&_col=content\", lambda row: row[\"text\"], \"sorted by text\"),\n    ],\n)\nasync def test_sortable(ds_client, query_string, sort_key, human_description_en):\n    path = f\"/fixtures/sortable.json?_shape=objects&_extra=human_description_en,next_url&{query_string}\"\n    fetched = []\n    page = 0\n    while path:\n        page += 1\n        assert page < 100\n        response = await ds_client.get(path)\n        assert human_description_en == response.json()[\"human_description_en\"]\n        fetched.extend(response.json()[\"rows\"])\n        path = response.json()[\"next_url\"]\n        if path:\n            path = path.replace(\"http://localhost\", \"\")\n    assert page == 5\n    expected = list(generate_sortable_rows(201))\n    expected.sort(key=sort_key)\n    assert [r[\"content\"] for r in expected] == [r[\"content\"] for r in fetched]\n\n\n@pytest.mark.asyncio\nasync def test_sortable_and_filtered(ds_client):\n    path = (\n        \"/fixtures/sortable.json\"\n        \"?content__contains=d&_sort_desc=sortable&_shape=objects\"\n        \"&_extra=human_description_en,count\"\n    )\n    response = await ds_client.get(path)\n    fetched = response.json()[\"rows\"]\n    assert (\n        'where content contains \"d\" sorted by sortable descending'\n        == response.json()[\"human_description_en\"]\n    )\n    expected = [row for row in generate_sortable_rows(201) if \"d\" in row[\"content\"]]\n    assert len(expected) == response.json()[\"count\"]\n    expected.sort(key=lambda row: -row[\"sortable\"])\n    assert [r[\"content\"] for r in expected] == [r[\"content\"] for r in fetched]\n\n\n@pytest.mark.asyncio\nasync def test_sortable_argument_errors(ds_client):\n    response = await ds_client.get(\"/fixtures/sortable.json?_sort=badcolumn\")\n    assert \"Cannot sort table by badcolumn\" == response.json()[\"error\"]\n    response = await ds_client.get(\"/fixtures/sortable.json?_sort_desc=badcolumn2\")\n    assert \"Cannot sort table by badcolumn2\" == response.json()[\"error\"]\n    response = await ds_client.get(\n        \"/fixtures/sortable.json?_sort=sortable_with_nulls&_sort_desc=sortable\"\n    )\n    assert (\n        \"Cannot use _sort and _sort_desc at the same time\" == response.json()[\"error\"]\n    )\n\n\n@pytest.mark.asyncio\nasync def test_sortable_columns_metadata(ds_client):\n    response = await ds_client.get(\"/fixtures/sortable.json?_sort=content\")\n    assert \"Cannot sort table by content\" == response.json()[\"error\"]\n    # no_primary_key has ALL sort options disabled\n    for column in (\"content\", \"a\", \"b\", \"c\"):\n        response = await ds_client.get(f\"/fixtures/sortable.json?_sort={column}\")\n        assert f\"Cannot sort table by {column}\" == response.json()[\"error\"]\n\n\n@pytest.mark.asyncio\n@pytest.mark.xfail\n@pytest.mark.parametrize(\n    \"path,expected_rows\",\n    [\n        (\n            \"/fixtures/searchable.json?_shape=arrays&_search=dog\",\n            [\n                [1, \"barry cat\", \"terry dog\", \"panther\"],\n                [2, \"terry dog\", \"sara weasel\", \"puma\"],\n            ],\n        ),\n        (\n            # Special keyword shouldn't break FTS query\n            \"/fixtures/searchable.json?_shape=arrays&_search=AND\",\n            [],\n        ),\n        (\n            # Without _searchmode=raw this should return no results\n            \"/fixtures/searchable.json?_shape=arrays&_search=te*+AND+do*\",\n            [],\n        ),\n        (\n            # _searchmode=raw\n            \"/fixtures/searchable.json?_shape=arrays&_search=te*+AND+do*&_searchmode=raw\",\n            [\n                [1, \"barry cat\", \"terry dog\", \"panther\"],\n                [2, \"terry dog\", \"sara weasel\", \"puma\"],\n            ],\n        ),\n        (\n            # _searchmode=raw combined with _search_COLUMN\n            \"/fixtures/searchable.json?_shape=arrays&_search_text2=te*&_searchmode=raw\",\n            [\n                [1, \"barry cat\", \"terry dog\", \"panther\"],\n            ],\n        ),\n        (\n            \"/fixtures/searchable.json?_shape=arrays&_search=weasel\",\n            [[2, \"terry dog\", \"sara weasel\", \"puma\"]],\n        ),\n        (\n            \"/fixtures/searchable.json?_shape=arrays&_search_text2=dog\",\n            [[1, \"barry cat\", \"terry dog\", \"panther\"]],\n        ),\n        (\n            \"/fixtures/searchable.json?_shape=arrays&_search_name%20with%20.%20and%20spaces=panther\",\n            [[1, \"barry cat\", \"terry dog\", \"panther\"]],\n        ),\n    ],\n)\nasync def test_searchable(ds_client, path, expected_rows):\n    response = await ds_client.get(path)\n    assert expected_rows == response.json()[\"rows\"]\n\n\n_SEARCHMODE_RAW_RESULTS = [\n    [1, \"barry cat\", \"terry dog\", \"panther\"],\n    [2, \"terry dog\", \"sara weasel\", \"puma\"],\n]\n\n\n@pytest.mark.parametrize(\n    \"table_metadata,querystring,expected_rows\",\n    [\n        (\n            {},\n            \"_search=te*+AND+do*\",\n            [],\n        ),\n        (\n            {\"searchmode\": \"raw\"},\n            \"_search=te*+AND+do*\",\n            _SEARCHMODE_RAW_RESULTS,\n        ),\n        (\n            {},\n            \"_search=te*+AND+do*&_searchmode=raw\",\n            _SEARCHMODE_RAW_RESULTS,\n        ),\n        # Can be over-ridden with _searchmode=escaped\n        (\n            {\"searchmode\": \"raw\"},\n            \"_search=te*+AND+do*&_searchmode=escaped\",\n            [],\n        ),\n    ],\n)\ndef test_searchmode(table_metadata, querystring, expected_rows):\n    with make_app_client(\n        metadata={\"databases\": {\"fixtures\": {\"tables\": {\"searchable\": table_metadata}}}}\n    ) as client:\n        response = client.get(\"/fixtures/searchable.json?_shape=arrays&\" + querystring)\n        assert expected_rows == response.json[\"rows\"]\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"path,expected_rows\",\n    [\n        (\n            \"/fixtures/searchable_view_configured_by_metadata.json?_shape=arrays&_search=weasel\",\n            [[2, \"terry dog\", \"sara weasel\", \"puma\"]],\n        ),\n        # This should return all results because search is not configured:\n        (\n            \"/fixtures/searchable_view.json?_shape=arrays&_search=weasel\",\n            [\n                [1, \"barry cat\", \"terry dog\", \"panther\"],\n                [2, \"terry dog\", \"sara weasel\", \"puma\"],\n            ],\n        ),\n        (\n            \"/fixtures/searchable_view.json?_shape=arrays&_search=weasel&_fts_table=searchable_fts&_fts_pk=pk\",\n            [[2, \"terry dog\", \"sara weasel\", \"puma\"]],\n        ),\n    ],\n)\nasync def test_searchable_views(ds_client, path, expected_rows):\n    response = await ds_client.get(path)\n    assert response.json()[\"rows\"] == expected_rows\n\n\n@pytest.mark.asyncio\nasync def test_searchable_invalid_column(ds_client):\n    response = await ds_client.get(\"/fixtures/searchable.json?_search_invalid=x\")\n    assert response.status_code == 400\n    assert response.json() == {\n        \"ok\": False,\n        \"error\": \"Cannot search by that column\",\n        \"status\": 400,\n        \"title\": None,\n    }\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"path,expected_rows\",\n    [\n        (\n            \"/fixtures/simple_primary_key.json?_shape=arrays&content=hello\",\n            [[1, \"hello\"]],\n        ),\n        (\n            \"/fixtures/simple_primary_key.json?_shape=arrays&content__contains=o\",\n            [\n                [1, \"hello\"],\n                [2, \"world\"],\n                [4, \"RENDER_CELL_DEMO\"],\n            ],\n        ),\n        (\n            \"/fixtures/simple_primary_key.json?_shape=arrays&content__exact=\",\n            [[3, \"\"]],\n        ),\n        (\n            \"/fixtures/simple_primary_key.json?_shape=arrays&content__not=world\",\n            [\n                [1, \"hello\"],\n                [3, \"\"],\n                [4, \"RENDER_CELL_DEMO\"],\n                [5, \"RENDER_CELL_ASYNC\"],\n            ],\n        ),\n    ],\n)\nasync def test_table_filter_queries(ds_client, path, expected_rows):\n    response = await ds_client.get(path)\n    assert response.json()[\"rows\"] == expected_rows\n\n\n@pytest.mark.asyncio\nasync def test_table_filter_queries_multiple_of_same_type(ds_client):\n    response = await ds_client.get(\n        \"/fixtures/simple_primary_key.json?_shape=arrays&content__not=world&content__not=hello\"\n    )\n    assert [\n        [3, \"\"],\n        [4, \"RENDER_CELL_DEMO\"],\n        [5, \"RENDER_CELL_ASYNC\"],\n    ] == response.json()[\"rows\"]\n\n\n@pytest.mark.skipif(not detect_json1(), reason=\"Requires the SQLite json1 module\")\n@pytest.mark.asyncio\nasync def test_table_filter_json_arraycontains(ds_client):\n    response = await ds_client.get(\n        \"/fixtures/facetable.json?_shape=arrays&tags__arraycontains=tag1\"\n    )\n    assert response.json()[\"rows\"] == [\n        [\n            1,\n            \"2019-01-14 08:00:00\",\n            1,\n            1,\n            \"CA\",\n            1,\n            \"Mission\",\n            '[\"tag1\", \"tag2\"]',\n            '[{\"foo\": \"bar\"}]',\n            \"one\",\n            \"n1\",\n        ],\n        [\n            2,\n            \"2019-01-14 08:00:00\",\n            1,\n            1,\n            \"CA\",\n            1,\n            \"Dogpatch\",\n            '[\"tag1\", \"tag3\"]',\n            \"[]\",\n            \"two\",\n            \"n2\",\n        ],\n    ]\n\n\n@pytest.mark.skipif(not detect_json1(), reason=\"Requires the SQLite json1 module\")\n@pytest.mark.asyncio\nasync def test_table_filter_json_arraynotcontains(ds_client):\n    response = await ds_client.get(\n        \"/fixtures/facetable.json?_shape=arrays&tags__arraynotcontains=tag3&tags__not=[]\"\n    )\n    assert response.json()[\"rows\"] == [\n        [\n            1,\n            \"2019-01-14 08:00:00\",\n            1,\n            1,\n            \"CA\",\n            1,\n            \"Mission\",\n            '[\"tag1\", \"tag2\"]',\n            '[{\"foo\": \"bar\"}]',\n            \"one\",\n            \"n1\",\n        ]\n    ]\n\n\n@pytest.mark.asyncio\nasync def test_table_filter_extra_where(ds_client):\n    response = await ds_client.get(\n        \"/fixtures/facetable.json?_shape=arrays&_where=_neighborhood='Dogpatch'\"\n    )\n    assert [\n        [\n            2,\n            \"2019-01-14 08:00:00\",\n            1,\n            1,\n            \"CA\",\n            1,\n            \"Dogpatch\",\n            '[\"tag1\", \"tag3\"]',\n            \"[]\",\n            \"two\",\n            \"n2\",\n        ]\n    ] == response.json()[\"rows\"]\n\n\n@pytest.mark.asyncio\nasync def test_table_filter_extra_where_invalid(ds_client):\n    response = await ds_client.get(\n        \"/fixtures/facetable.json?_where=_neighborhood=Dogpatch'\"\n    )\n    assert response.status_code == 400\n    assert \"Invalid SQL\" == response.json()[\"title\"]\n\n\ndef test_table_filter_extra_where_disabled_if_no_sql_allowed():\n    with make_app_client(config={\"allow_sql\": {}}) as client:\n        response = client.get(\n            \"/fixtures/facetable.json?_where=_neighborhood='Dogpatch'\"\n        )\n        assert response.status_code == 403\n        assert \"_where= is not allowed\" == response.json[\"error\"]\n\n\n@pytest.mark.asyncio\nasync def test_table_through(ds_client):\n    # Just the museums:\n    response = await ds_client.get(\n        \"/fixtures/roadside_attractions.json?_shape=arrays\"\n        '&_through={\"table\":\"roadside_attraction_characteristics\",\"column\":\"characteristic_id\",\"value\":\"1\"}'\n        \"&_extra=human_description_en\"\n    )\n    assert response.json()[\"rows\"] == [\n        [\n            3,\n            \"Burlingame Museum of PEZ Memorabilia\",\n            \"214 California Drive, Burlingame, CA 94010\",\n            None,\n            37.5793,\n            -122.3442,\n        ],\n        [\n            4,\n            \"Bigfoot Discovery Museum\",\n            \"5497 Highway 9, Felton, CA 95018\",\n            \"https://www.bigfootdiscoveryproject.com/\",\n            37.0414,\n            -122.0725,\n        ],\n    ]\n\n    assert (\n        response.json()[\"human_description_en\"]\n        == 'where roadside_attraction_characteristics.characteristic_id = \"1\"'\n    )\n\n\n@pytest.mark.asyncio\nasync def test_max_returned_rows(ds_client):\n    response = await ds_client.get(\n        \"/fixtures/-/query.json?sql=select+content+from+no_primary_key\"\n    )\n    data = response.json()\n    assert data[\"truncated\"]\n    assert 100 == len(data[\"rows\"])\n\n\n@pytest.mark.asyncio\nasync def test_view(ds_client):\n    response = await ds_client.get(\"/fixtures/simple_view.json?_shape=objects\")\n    assert response.status_code == 200\n    data = response.json()\n    assert data[\"rows\"] == [\n        {\"upper_content\": \"HELLO\", \"content\": \"hello\"},\n        {\"upper_content\": \"WORLD\", \"content\": \"world\"},\n        {\"upper_content\": \"\", \"content\": \"\"},\n        {\"upper_content\": \"RENDER_CELL_DEMO\", \"content\": \"RENDER_CELL_DEMO\"},\n        {\"upper_content\": \"RENDER_CELL_ASYNC\", \"content\": \"RENDER_CELL_ASYNC\"},\n    ]\n\n\ndef test_page_size_matching_max_returned_rows(\n    app_client_returned_rows_matches_page_size,\n):\n    fetched = []\n    path = \"/fixtures/no_primary_key.json?_extra=next_url\"\n    while path:\n        response = app_client_returned_rows_matches_page_size.get(path)\n        fetched.extend(response.json[\"rows\"])\n        assert len(response.json[\"rows\"]) in (2, 50)\n        path = response.json[\"next_url\"]\n        if path:\n            path = path.replace(\"http://localhost\", \"\")\n    assert len(fetched) == 202\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"path,expected_facet_results\",\n    [\n        (\n            \"/fixtures/facetable.json?_facet=state&_facet=_city_id\",\n            {\n                \"state\": {\n                    \"name\": \"state\",\n                    \"hideable\": True,\n                    \"type\": \"column\",\n                    \"toggle_url\": \"/fixtures/facetable.json?_facet=_city_id\",\n                    \"results\": [\n                        {\n                            \"value\": \"CA\",\n                            \"label\": \"CA\",\n                            \"count\": 10,\n                            \"toggle_url\": \"_facet=state&_facet=_city_id&state=CA\",\n                            \"selected\": False,\n                        },\n                        {\n                            \"value\": \"MI\",\n                            \"label\": \"MI\",\n                            \"count\": 4,\n                            \"toggle_url\": \"_facet=state&_facet=_city_id&state=MI\",\n                            \"selected\": False,\n                        },\n                        {\n                            \"value\": \"MC\",\n                            \"label\": \"MC\",\n                            \"count\": 1,\n                            \"toggle_url\": \"_facet=state&_facet=_city_id&state=MC\",\n                            \"selected\": False,\n                        },\n                    ],\n                    \"truncated\": False,\n                },\n                \"_city_id\": {\n                    \"name\": \"_city_id\",\n                    \"hideable\": True,\n                    \"type\": \"column\",\n                    \"toggle_url\": \"/fixtures/facetable.json?_facet=state\",\n                    \"results\": [\n                        {\n                            \"value\": 1,\n                            \"label\": \"San Francisco\",\n                            \"count\": 6,\n                            \"toggle_url\": \"_facet=state&_facet=_city_id&_city_id__exact=1\",\n                            \"selected\": False,\n                        },\n                        {\n                            \"value\": 2,\n                            \"label\": \"Los Angeles\",\n                            \"count\": 4,\n                            \"toggle_url\": \"_facet=state&_facet=_city_id&_city_id__exact=2\",\n                            \"selected\": False,\n                        },\n                        {\n                            \"value\": 3,\n                            \"label\": \"Detroit\",\n                            \"count\": 4,\n                            \"toggle_url\": \"_facet=state&_facet=_city_id&_city_id__exact=3\",\n                            \"selected\": False,\n                        },\n                        {\n                            \"value\": 4,\n                            \"label\": \"Memnonia\",\n                            \"count\": 1,\n                            \"toggle_url\": \"_facet=state&_facet=_city_id&_city_id__exact=4\",\n                            \"selected\": False,\n                        },\n                    ],\n                    \"truncated\": False,\n                },\n            },\n        ),\n        (\n            \"/fixtures/facetable.json?_facet=state&_facet=_city_id&state=MI\",\n            {\n                \"state\": {\n                    \"name\": \"state\",\n                    \"hideable\": True,\n                    \"type\": \"column\",\n                    \"toggle_url\": \"/fixtures/facetable.json?_facet=_city_id&state=MI\",\n                    \"results\": [\n                        {\n                            \"value\": \"MI\",\n                            \"label\": \"MI\",\n                            \"count\": 4,\n                            \"selected\": True,\n                            \"toggle_url\": \"_facet=state&_facet=_city_id\",\n                        }\n                    ],\n                    \"truncated\": False,\n                },\n                \"_city_id\": {\n                    \"name\": \"_city_id\",\n                    \"hideable\": True,\n                    \"type\": \"column\",\n                    \"toggle_url\": \"/fixtures/facetable.json?_facet=state&state=MI\",\n                    \"results\": [\n                        {\n                            \"value\": 3,\n                            \"label\": \"Detroit\",\n                            \"count\": 4,\n                            \"selected\": False,\n                            \"toggle_url\": \"_facet=state&_facet=_city_id&state=MI&_city_id__exact=3\",\n                        }\n                    ],\n                    \"truncated\": False,\n                },\n            },\n        ),\n        (\n            \"/fixtures/facetable.json?_facet=planet_int\",\n            {\n                \"planet_int\": {\n                    \"name\": \"planet_int\",\n                    \"hideable\": True,\n                    \"type\": \"column\",\n                    \"toggle_url\": \"/fixtures/facetable.json\",\n                    \"results\": [\n                        {\n                            \"value\": 1,\n                            \"label\": 1,\n                            \"count\": 14,\n                            \"selected\": False,\n                            \"toggle_url\": \"_facet=planet_int&planet_int=1\",\n                        },\n                        {\n                            \"value\": 2,\n                            \"label\": 2,\n                            \"count\": 1,\n                            \"selected\": False,\n                            \"toggle_url\": \"_facet=planet_int&planet_int=2\",\n                        },\n                    ],\n                    \"truncated\": False,\n                }\n            },\n        ),\n        (\n            # planet_int is an integer field:\n            \"/fixtures/facetable.json?_facet=planet_int&planet_int=1\",\n            {\n                \"planet_int\": {\n                    \"name\": \"planet_int\",\n                    \"hideable\": True,\n                    \"type\": \"column\",\n                    \"toggle_url\": \"/fixtures/facetable.json?planet_int=1\",\n                    \"results\": [\n                        {\n                            \"value\": 1,\n                            \"label\": 1,\n                            \"count\": 14,\n                            \"selected\": True,\n                            \"toggle_url\": \"_facet=planet_int\",\n                        }\n                    ],\n                    \"truncated\": False,\n                }\n            },\n        ),\n    ],\n)\nasync def test_facets(ds_client, path, expected_facet_results):\n    response = await ds_client.get(path)\n    facet_results = response.json()[\"facet_results\"]\n    # We only compare the querystring portion of the taggle_url\n    for facet_name, facet_info in facet_results[\"results\"].items():\n        assert facet_name == facet_info[\"name\"]\n        assert False is facet_info[\"truncated\"]\n        for facet_value in facet_info[\"results\"]:\n            facet_value[\"toggle_url\"] = facet_value[\"toggle_url\"].split(\"?\")[1]\n    assert expected_facet_results == facet_results[\"results\"]\n\n\n@pytest.mark.asyncio\n@pytest.mark.skipif(not detect_json1(), reason=\"requires JSON1 extension\")\nasync def test_facets_array(ds_client):\n    response = await ds_client.get(\"/fixtures/facetable.json?_facet_array=tags\")\n    facet_results = response.json()[\"facet_results\"]\n    assert facet_results[\"results\"][\"tags\"][\"results\"] == [\n        {\n            \"value\": \"tag1\",\n            \"label\": \"tag1\",\n            \"count\": 2,\n            \"toggle_url\": \"http://localhost/fixtures/facetable.json?_facet_array=tags&tags__arraycontains=tag1\",\n            \"selected\": False,\n        },\n        {\n            \"value\": \"tag2\",\n            \"label\": \"tag2\",\n            \"count\": 1,\n            \"toggle_url\": \"http://localhost/fixtures/facetable.json?_facet_array=tags&tags__arraycontains=tag2\",\n            \"selected\": False,\n        },\n        {\n            \"value\": \"tag3\",\n            \"label\": \"tag3\",\n            \"count\": 1,\n            \"toggle_url\": \"http://localhost/fixtures/facetable.json?_facet_array=tags&tags__arraycontains=tag3\",\n            \"selected\": False,\n        },\n    ]\n\n\n@pytest.mark.asyncio\nasync def test_suggested_facets(ds_client):\n    suggestions = [\n        {\n            \"name\": suggestion[\"name\"],\n            \"querystring\": suggestion[\"toggle_url\"].split(\"?\")[-1],\n        }\n        for suggestion in (\n            await ds_client.get(\"/fixtures/facetable.json?_extra=suggested_facets\")\n        ).json()[\"suggested_facets\"]\n    ]\n    expected = [\n        {\"name\": \"created\", \"querystring\": \"_extra=suggested_facets&_facet=created\"},\n        {\n            \"name\": \"planet_int\",\n            \"querystring\": \"_extra=suggested_facets&_facet=planet_int\",\n        },\n        {\"name\": \"on_earth\", \"querystring\": \"_extra=suggested_facets&_facet=on_earth\"},\n        {\"name\": \"state\", \"querystring\": \"_extra=suggested_facets&_facet=state\"},\n        {\"name\": \"_city_id\", \"querystring\": \"_extra=suggested_facets&_facet=_city_id\"},\n        {\n            \"name\": \"_neighborhood\",\n            \"querystring\": \"_extra=suggested_facets&_facet=_neighborhood\",\n        },\n        {\"name\": \"tags\", \"querystring\": \"_extra=suggested_facets&_facet=tags\"},\n        {\n            \"name\": \"complex_array\",\n            \"querystring\": \"_extra=suggested_facets&_facet=complex_array\",\n        },\n        {\n            \"name\": \"created\",\n            \"querystring\": \"_extra=suggested_facets&_facet_date=created\",\n        },\n    ]\n    if detect_json1():\n        expected.append(\n            {\"name\": \"tags\", \"querystring\": \"_extra=suggested_facets&_facet_array=tags\"}\n        )\n    assert expected == suggestions\n\n\ndef test_allow_facet_off():\n    with make_app_client(settings={\"allow_facet\": False}) as client:\n        assert (\n            client.get(\n                \"/fixtures/facetable.json?_facet=planet_int&_extra=suggested_facets\"\n            ).status\n            == 400\n        )\n        data = client.get(\"/fixtures/facetable.json?_extra=suggested_facets\").json\n        # Should not suggest any facets either:\n        assert [] == data[\"suggested_facets\"]\n\n\ndef test_suggest_facets_off():\n    with make_app_client(settings={\"suggest_facets\": False}) as client:\n        # Now suggested_facets should be []\n        assert (\n            []\n            == client.get(\"/fixtures/facetable.json?_extra=suggested_facets\").json[\n                \"suggested_facets\"\n            ]\n        )\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\"nofacet\", (True, False))\nasync def test_nofacet(ds_client, nofacet):\n    path = \"/fixtures/facetable.json?_facet=state&_extra=suggested_facets\"\n    if nofacet:\n        path += \"&_nofacet=1\"\n    response = await ds_client.get(path)\n    if nofacet:\n        assert response.json()[\"suggested_facets\"] == []\n        assert response.json()[\"facet_results\"][\"results\"] == {}\n    else:\n        assert response.json()[\"suggested_facets\"] != []\n        assert response.json()[\"facet_results\"][\"results\"] != {}\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\"nosuggest\", (True, False))\nasync def test_nosuggest(ds_client, nosuggest):\n    path = \"/fixtures/facetable.json?_facet=state&_extra=suggested_facets\"\n    if nosuggest:\n        path += \"&_nosuggest=1\"\n    response = await ds_client.get(path)\n    if nosuggest:\n        assert response.json()[\"suggested_facets\"] == []\n        # But facets should still be returned:\n        assert response.json()[\"facet_results\"] != {}\n    else:\n        assert response.json()[\"suggested_facets\"] != []\n        assert response.json()[\"facet_results\"] != {}\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\"nocount,expected_count\", ((True, None), (False, 15)))\nasync def test_nocount(ds_client, nocount, expected_count):\n    path = \"/fixtures/facetable.json?_extra=count\"\n    if nocount:\n        path += \"&_nocount=1\"\n    response = await ds_client.get(path)\n    assert response.json()[\"count\"] == expected_count\n\n\ndef test_nocount_nofacet_if_shape_is_object(app_client_with_trace):\n    response = app_client_with_trace.get(\n        \"/fixtures/facetable.json?_trace=1&_shape=object\"\n    )\n    assert \"count(*)\" not in response.text\n\n\n@pytest.mark.asyncio\nasync def test_expand_labels(ds_client):\n    response = await ds_client.get(\n        \"/fixtures/facetable.json?_shape=object&_labels=1&_size=2\"\n        \"&_neighborhood__contains=c\"\n    )\n    assert response.json() == {\n        \"2\": {\n            \"pk\": 2,\n            \"created\": \"2019-01-14 08:00:00\",\n            \"planet_int\": 1,\n            \"on_earth\": 1,\n            \"state\": \"CA\",\n            \"_city_id\": {\"value\": 1, \"label\": \"San Francisco\"},\n            \"_neighborhood\": \"Dogpatch\",\n            \"tags\": '[\"tag1\", \"tag3\"]',\n            \"complex_array\": \"[]\",\n            \"distinct_some_null\": \"two\",\n            \"n\": \"n2\",\n        },\n        \"13\": {\n            \"pk\": 13,\n            \"created\": \"2019-01-17 08:00:00\",\n            \"planet_int\": 1,\n            \"on_earth\": 1,\n            \"state\": \"MI\",\n            \"_city_id\": {\"value\": 3, \"label\": \"Detroit\"},\n            \"_neighborhood\": \"Corktown\",\n            \"tags\": \"[]\",\n            \"complex_array\": \"[]\",\n            \"distinct_some_null\": None,\n            \"n\": None,\n        },\n    }\n\n\n@pytest.mark.asyncio\nasync def test_expand_label(ds_client):\n    response = await ds_client.get(\n        \"/fixtures/foreign_key_references.json?_shape=object\"\n        \"&_label=foreign_key_with_label&_size=1\"\n    )\n    assert response.json() == {\n        \"1\": {\n            \"pk\": \"1\",\n            \"foreign_key_with_label\": {\"value\": 1, \"label\": \"hello\"},\n            \"foreign_key_with_blank_label\": 3,\n            \"foreign_key_with_no_label\": \"1\",\n            \"foreign_key_compound_pk1\": \"a\",\n            \"foreign_key_compound_pk2\": \"b\",\n        }\n    }\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"path,expected_cache_control\",\n    [\n        (\"/fixtures/facetable.json\", \"max-age=5\"),\n        (\"/fixtures/facetable.json?_ttl=invalid\", \"max-age=5\"),\n        (\"/fixtures/facetable.json?_ttl=10\", \"max-age=10\"),\n        (\"/fixtures/facetable.json?_ttl=0\", \"no-cache\"),\n    ],\n)\nasync def test_ttl_parameter(ds_client, path, expected_cache_control):\n    response = await ds_client.get(path)\n    assert response.headers[\"Cache-Control\"] == expected_cache_control\n\n\n@pytest.mark.asyncio\nasync def test_infinity_returned_as_null(ds_client):\n    response = await ds_client.get(\"/fixtures/infinity.json?_shape=array\")\n    assert response.json() == [\n        {\"rowid\": 1, \"value\": None},\n        {\"rowid\": 2, \"value\": None},\n        {\"rowid\": 3, \"value\": 1.5},\n    ]\n\n\n@pytest.mark.asyncio\nasync def test_infinity_returned_as_invalid_json_if_requested(ds_client):\n    response = await ds_client.get(\n        \"/fixtures/infinity.json?_shape=array&_json_infinity=1\"\n    )\n    assert response.json() == [\n        {\"rowid\": 1, \"value\": float(\"inf\")},\n        {\"rowid\": 2, \"value\": float(\"-inf\")},\n        {\"rowid\": 3, \"value\": 1.5},\n    ]\n\n\n@pytest.mark.asyncio\nasync def test_custom_query_with_unicode_characters(ds_client):\n    # /fixtures/𝐜𝐢𝐭𝐢𝐞𝐬.json\n    response = await ds_client.get(\n        \"/fixtures/~F0~9D~90~9C~F0~9D~90~A2~F0~9D~90~AD~F0~9D~90~A2~F0~9D~90~9E~F0~9D~90~AC.json?_shape=array\"\n    )\n    assert response.json() == [{\"id\": 1, \"name\": \"San Francisco\"}]\n\n\n@pytest.mark.asyncio\nasync def test_null_and_compound_foreign_keys_are_not_expanded(ds_client):\n    response = await ds_client.get(\n        \"/fixtures/foreign_key_references.json?_shape=array&_labels=on\"\n    )\n    assert response.json() == [\n        {\n            \"pk\": \"1\",\n            \"foreign_key_with_label\": {\"value\": 1, \"label\": \"hello\"},\n            \"foreign_key_with_blank_label\": {\"value\": 3, \"label\": \"\"},\n            \"foreign_key_with_no_label\": {\"value\": \"1\", \"label\": \"1\"},\n            \"foreign_key_compound_pk1\": \"a\",\n            \"foreign_key_compound_pk2\": \"b\",\n        },\n        {\n            \"pk\": \"2\",\n            \"foreign_key_with_label\": None,\n            \"foreign_key_with_blank_label\": None,\n            \"foreign_key_with_no_label\": None,\n            \"foreign_key_compound_pk1\": None,\n            \"foreign_key_compound_pk2\": None,\n        },\n    ]\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"path,expected_json,expected_text\",\n    [\n        (\n            \"/fixtures/binary_data.json?_shape=array\",\n            [\n                {\"rowid\": 1, \"data\": {\"$base64\": True, \"encoded\": \"FRwCx60F/g==\"}},\n                {\"rowid\": 2, \"data\": {\"$base64\": True, \"encoded\": \"FRwDx60F/g==\"}},\n                {\"rowid\": 3, \"data\": None},\n            ],\n            None,\n        ),\n        (\n            \"/fixtures/binary_data.json?_shape=array&_nl=on\",\n            None,\n            (\n                '{\"rowid\": 1, \"data\": {\"$base64\": true, \"encoded\": \"FRwCx60F/g==\"}}\\n'\n                '{\"rowid\": 2, \"data\": {\"$base64\": true, \"encoded\": \"FRwDx60F/g==\"}}\\n'\n                '{\"rowid\": 3, \"data\": null}'\n            ),\n        ),\n    ],\n)\nasync def test_binary_data_in_json(ds_client, path, expected_json, expected_text):\n    response = await ds_client.get(path)\n    if expected_json:\n        assert response.json() == expected_json\n    else:\n        assert response.text == expected_text\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"qs\",\n    [\n        \"\",\n        \"?_shape=arrays\",\n        \"?_shape=arrayfirst\",\n        \"?_shape=object\",\n        \"?_shape=objects\",\n        \"?_shape=array\",\n        \"?_shape=array&_nl=on\",\n    ],\n)\nasync def test_paginate_using_link_header(ds_client, qs):\n    path = f\"/fixtures/compound_three_primary_keys.json{qs}\"\n    num_pages = 0\n    while path:\n        response = await ds_client.get(path)\n        assert response.status_code == 200\n        num_pages += 1\n        link = response.headers.get(\"link\")\n        if link:\n            assert link.startswith(\"<\")\n            assert link.endswith('>; rel=\"next\"')\n            path = link[1:].split(\">\")[0]\n            path = path.replace(\"http://localhost\", \"\")\n        else:\n            path = None\n    assert num_pages == 21\n\n\n@pytest.mark.skipif(\n    sqlite_version() < (3, 31, 0),\n    reason=\"generated columns were added in SQLite 3.31.0\",\n)\ndef test_generated_columns_are_visible_in_datasette():\n    with make_app_client(extra_databases={\"generated.db\": \"\"\"\n                CREATE TABLE generated_columns (\n                    body TEXT,\n                    id INT GENERATED ALWAYS AS (json_extract(body, '$.number')) STORED,\n                    consideration INT GENERATED ALWAYS AS (json_extract(body, '$.string')) STORED\n                );\n                INSERT INTO generated_columns (body) VALUES (\n                    '{\"number\": 1, \"string\": \"This is a string\"}'\n                );\"\"\"}) as client:\n        response = client.get(\"/generated/generated_columns.json?_shape=array\")\n        assert response.json == [\n            {\n                \"rowid\": 1,\n                \"body\": '{\"number\": 1, \"string\": \"This is a string\"}',\n                \"id\": 1,\n                \"consideration\": \"This is a string\",\n            }\n        ]\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"path,expected_columns\",\n    (\n        (\"/fixtures/facetable.json?_col=created\", [\"pk\", \"created\"]),\n        (\n            \"/fixtures/facetable.json?_nocol=created\",\n            [\n                \"pk\",\n                \"planet_int\",\n                \"on_earth\",\n                \"state\",\n                \"_city_id\",\n                \"_neighborhood\",\n                \"tags\",\n                \"complex_array\",\n                \"distinct_some_null\",\n                \"n\",\n            ],\n        ),\n        (\n            \"/fixtures/facetable.json?_col=state&_col=created\",\n            [\"pk\", \"state\", \"created\"],\n        ),\n        (\n            \"/fixtures/facetable.json?_col=state&_col=state\",\n            [\"pk\", \"state\"],\n        ),\n        (\n            \"/fixtures/facetable.json?_col=state&_col=created&_nocol=created\",\n            [\"pk\", \"state\"],\n        ),\n        (\n            # Ensure faceting doesn't break, https://github.com/simonw/datasette/issues/1345\n            \"/fixtures/facetable.json?_nocol=state&_facet=state\",\n            [\n                \"pk\",\n                \"created\",\n                \"planet_int\",\n                \"on_earth\",\n                \"_city_id\",\n                \"_neighborhood\",\n                \"tags\",\n                \"complex_array\",\n                \"distinct_some_null\",\n                \"n\",\n            ],\n        ),\n        (\n            \"/fixtures/simple_view.json?_nocol=content\",\n            [\"upper_content\"],\n        ),\n        (\"/fixtures/simple_view.json?_col=content\", [\"content\"]),\n    ),\n)\nasync def test_col_nocol(ds_client, path, expected_columns):\n    response = await ds_client.get(path + \"&_extra=columns\")\n    assert response.status_code == 200\n    columns = response.json()[\"columns\"]\n    assert columns == expected_columns\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"path,expected_error\",\n    (\n        (\"/fixtures/facetable.json?_col=bad\", \"_col=bad - invalid columns\"),\n        (\"/fixtures/facetable.json?_nocol=bad\", \"_nocol=bad - invalid columns\"),\n        (\"/fixtures/facetable.json?_nocol=pk\", \"_nocol=pk - invalid columns\"),\n        (\"/fixtures/simple_view.json?_col=bad\", \"_col=bad - invalid columns\"),\n    ),\n)\nasync def test_col_nocol_errors(ds_client, path, expected_error):\n    response = await ds_client.get(path)\n    assert response.status_code == 400\n    assert response.json()[\"error\"] == expected_error\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"extra,expected_json\",\n    (\n        (\n            \"columns\",\n            {\n                \"ok\": True,\n                \"next\": None,\n                \"columns\": [\"id\", \"content\", \"content2\"],\n                \"rows\": [{\"id\": \"1\", \"content\": \"hey\", \"content2\": \"world\"}],\n                \"truncated\": False,\n            },\n        ),\n        (\n            \"count\",\n            {\n                \"ok\": True,\n                \"next\": None,\n                \"rows\": [{\"id\": \"1\", \"content\": \"hey\", \"content2\": \"world\"}],\n                \"truncated\": False,\n                \"count\": 1,\n            },\n        ),\n    ),\n)\nasync def test_table_extras(ds_client, extra, expected_json):\n    response = await ds_client.get(\n        \"/fixtures/primary_key_multiple_columns.json?_extra=\" + extra\n    )\n    assert response.status_code == 200\n    assert response.json() == expected_json\n\n\n@pytest.mark.asyncio\nasync def test_extra_render_cell():\n    \"\"\"Test that _extra=render_cell returns rendered HTML from render_cell plugin hook\"\"\"\n    from datasette import hookimpl\n    from datasette.app import Datasette\n\n    class TestRenderCellPlugin:\n        __name__ = \"TestRenderCellPlugin\"\n\n        @hookimpl\n        def render_cell(self, value, column, table, database):\n            # Only modify cells in our test table\n            if table == \"test_render\" and column == \"name\":\n                return f\"{value}\"\n            return None\n\n    ds = Datasette(memory=True)\n    await ds.invoke_startup()\n    db = ds.add_memory_database(\"test_table_render\")\n    await db.execute_write(\n        \"create table test_render (id integer primary key, name text)\"\n    )\n    await db.execute_write(\"insert into test_render values (1, 'Alice')\")\n    await db.execute_write(\"insert into test_render values (2, 'Bob')\")\n\n    # Register our test plugin\n    ds.pm.register(TestRenderCellPlugin(), name=\"TestRenderCellPlugin\")\n\n    try:\n        # Request with _extra=render_cell\n        response = await ds.client.get(\n            \"/test_table_render/test_render.json?_extra=render_cell\"\n        )\n        assert response.status_code == 200\n        data = response.json()\n\n        # Verify the response structure\n        assert \"render_cell\" in data\n        assert \"rows\" in data\n\n        # render_cell should be a list of rows, each row being a dict of column -> rendered HTML\n        # Only columns modified by plugins are included (sparse output)\n        render_cell = data[\"render_cell\"]\n        assert len(render_cell) == 2\n\n        # First row: id=1, name='Alice'\n        # The 'name' column should be rendered by our plugin as Alice\n        assert render_cell[0][\"name\"] == \"Alice\"\n        # The 'id' column is not included since no plugin modified it\n        assert \"id\" not in render_cell[0]\n\n        # Second row: id=2, name='Bob'\n        assert render_cell[1][\"name\"] == \"Bob\"\n        assert \"id\" not in render_cell[1]\n\n        # The regular rows should still contain raw values\n        assert data[\"rows\"] == [\n            {\"id\": 1, \"name\": \"Alice\"},\n            {\"id\": 2, \"name\": \"Bob\"},\n        ]\n\n    finally:\n        ds.pm.unregister(name=\"TestRenderCellPlugin\")\n"
  },
  {
    "path": "tests/test_table_html.py",
    "content": "from datasette.app import Datasette\nfrom bs4 import BeautifulSoup as Soup\nfrom .fixtures import make_app_client\nimport pathlib\nimport pytest\nimport urllib.parse\nfrom .utils import inner_html\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"path,expected_definition_sql\",\n    [\n        (\n            \"/fixtures/facet_cities\",\n            \"\"\"\nCREATE TABLE facet_cities (\n    id integer primary key,\n    name text\n);\n        \"\"\".strip(),\n        ),\n        (\n            \"/fixtures/compound_three_primary_keys\",\n            \"\"\"\nCREATE TABLE compound_three_primary_keys (\n  pk1 varchar(30),\n  pk2 varchar(30),\n  pk3 varchar(30),\n  content text,\n  PRIMARY KEY (pk1, pk2, pk3)\n);\nCREATE INDEX idx_compound_three_primary_keys_content ON compound_three_primary_keys(content);\n            \"\"\".strip(),\n        ),\n    ],\n)\nasync def test_table_definition_sql(path, expected_definition_sql, ds_client):\n    response = await ds_client.get(path)\n    pre = Soup(response.text, \"html.parser\").select_one(\"pre.wrapped-sql\")\n    assert expected_definition_sql == pre.string\n\n\ndef test_table_cell_truncation():\n    with make_app_client(settings={\"truncate_cells_html\": 5}) as client:\n        response = client.get(\"/fixtures/facetable\")\n        assert response.status == 200\n        table = Soup(response.body, \"html.parser\").find(\"table\")\n        assert table[\"class\"] == [\"rows-and-columns\"]\n        assert [\n            \"Missi…\",\n            \"Dogpa…\",\n            \"SOMA\",\n            \"Tende…\",\n            \"Berna…\",\n            \"Hayes…\",\n            \"Holly…\",\n            \"Downt…\",\n            \"Los F…\",\n            \"Korea…\",\n            \"Downt…\",\n            \"Greek…\",\n            \"Corkt…\",\n            \"Mexic…\",\n            \"Arcad…\",\n        ] == [\n            td.string\n            for td in table.find_all(\"td\", {\"class\": \"col-neighborhood-b352a7\"})\n        ]\n        # URLs should be truncated too\n        response2 = client.get(\"/fixtures/roadside_attractions\")\n        assert response2.status == 200\n        table = Soup(response2.body, \"html.parser\").find(\"table\")\n        tds = table.find_all(\"td\", {\"class\": \"col-url\"})\n        assert [str(td) for td in tds] == [\n            '
    ',\n            '',\n            '',\n            '',\n        ]\n\n\n@pytest.mark.asyncio\nasync def test_add_filter_redirects(ds_client):\n    filter_args = urllib.parse.urlencode(\n        {\"_filter_column\": \"content\", \"_filter_op\": \"startswith\", \"_filter_value\": \"x\"}\n    )\n    path_base = \"/fixtures/simple_primary_key\"\n    path = path_base + \"?\" + filter_args\n    response = await ds_client.get(path)\n    assert response.status_code == 302\n    assert response.headers[\"Location\"].endswith(\"?content__startswith=x\")\n\n    # Adding a redirect to an existing query string:\n    path = path_base + \"?foo=bar&\" + filter_args\n    response = await ds_client.get(path)\n    assert response.status_code == 302\n    assert response.headers[\"Location\"].endswith(\"?foo=bar&content__startswith=x\")\n\n    # Test that op with a __x suffix overrides the filter value\n    path = (\n        path_base\n        + \"?\"\n        + urllib.parse.urlencode(\n            {\n                \"_filter_column\": \"content\",\n                \"_filter_op\": \"isnull__5\",\n                \"_filter_value\": \"x\",\n            }\n        )\n    )\n    response = await ds_client.get(path)\n    assert response.status_code == 302\n    assert response.headers[\"Location\"].endswith(\"?content__isnull=5\")\n\n\n@pytest.mark.asyncio\nasync def test_existing_filter_redirects(ds_client):\n    filter_args = {\n        \"_filter_column_1\": \"name\",\n        \"_filter_op_1\": \"contains\",\n        \"_filter_value_1\": \"hello\",\n        \"_filter_column_2\": \"age\",\n        \"_filter_op_2\": \"gte\",\n        \"_filter_value_2\": \"22\",\n        \"_filter_column_3\": \"age\",\n        \"_filter_op_3\": \"lt\",\n        \"_filter_value_3\": \"30\",\n        \"_filter_column_4\": \"name\",\n        \"_filter_op_4\": \"contains\",\n        \"_filter_value_4\": \"world\",\n    }\n    path_base = \"/fixtures/simple_primary_key\"\n    path = path_base + \"?\" + urllib.parse.urlencode(filter_args)\n    response = await ds_client.get(path)\n    assert response.status_code == 302\n    assert_querystring_equal(\n        \"name__contains=hello&age__gte=22&age__lt=30&name__contains=world\",\n        response.headers[\"Location\"].split(\"?\")[1],\n    )\n\n    # Setting _filter_column_3 to empty string should remove *_3 entirely\n    filter_args[\"_filter_column_3\"] = \"\"\n    path = path_base + \"?\" + urllib.parse.urlencode(filter_args)\n    response = await ds_client.get(path)\n    assert response.status_code == 302\n    assert_querystring_equal(\n        \"name__contains=hello&age__gte=22&name__contains=world\",\n        response.headers[\"Location\"].split(\"?\")[1],\n    )\n\n    # ?_filter_op=exact should be removed if unaccompanied by _fiter_column\n    response = await ds_client.get(path_base + \"?_filter_op=exact\")\n    assert response.status_code == 302\n    assert \"?\" not in response.headers[\"Location\"]\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"qs,expected_hidden\",\n    (\n        # Things that should be reflected in hidden form fields:\n        (\"_facet=_neighborhood\", {\"_facet\": \"_neighborhood\"}),\n        (\"_where=1+=+1&_col=_city_id\", {\"_where\": \"1 = 1\", \"_col\": \"_city_id\"}),\n        # Things that should NOT be reflected in hidden form fields:\n        (\n            \"_facet=_neighborhood&_neighborhood__exact=Downtown\",\n            {\"_facet\": \"_neighborhood\"},\n        ),\n        (\"_facet=_neighborhood&_city_id__gt=1\", {\"_facet\": \"_neighborhood\"}),\n    ),\n)\nasync def test_reflected_hidden_form_fields(ds_client, qs, expected_hidden):\n    # https://github.com/simonw/datasette/issues/1527\n    response = await ds_client.get(\"/fixtures/facetable?{}\".format(qs))\n    # In this case we should NOT have a hidden _neighborhood__exact=Downtown field\n    form = Soup(response.text, \"html.parser\").find(\"form\")\n    hidden_inputs = {\n        input[\"name\"]: input[\"value\"] for input in form.select(\"input[type=hidden]\")\n    }\n    assert hidden_inputs == expected_hidden\n\n\n@pytest.mark.asyncio\nasync def test_empty_search_parameter_gets_removed(ds_client):\n    path_base = \"/fixtures/simple_primary_key\"\n    path = (\n        path_base\n        + \"?\"\n        + urllib.parse.urlencode(\n            {\n                \"_search\": \"\",\n                \"_filter_column\": \"name\",\n                \"_filter_op\": \"exact\",\n                \"_filter_value\": \"chidi\",\n            }\n        )\n    )\n    response = await ds_client.get(path)\n    assert response.status_code == 302\n    assert response.headers[\"Location\"].endswith(\"?name__exact=chidi\")\n\n\n@pytest.mark.asyncio\nasync def test_searchable_view_persists_fts_table(ds_client):\n    # The search form should persist ?_fts_table as a hidden field\n    response = await ds_client.get(\n        \"/fixtures/searchable_view?_fts_table=searchable_fts&_fts_pk=pk\"\n    )\n    inputs = Soup(response.text, \"html.parser\").find(\"form\").find_all(\"input\")\n    hiddens = [i for i in inputs if i[\"type\"] == \"hidden\"]\n    assert [(\"_fts_table\", \"searchable_fts\"), (\"_fts_pk\", \"pk\")] == [\n        (hidden[\"name\"], hidden[\"value\"]) for hidden in hiddens\n    ]\n\n\n@pytest.mark.asyncio\nasync def test_sort_by_desc_redirects(ds_client):\n    path_base = \"/fixtures/sortable\"\n    path = (\n        path_base\n        + \"?\"\n        + urllib.parse.urlencode({\"_sort\": \"sortable\", \"_sort_by_desc\": \"1\"})\n    )\n    response = await ds_client.get(path)\n    assert response.status_code == 302\n    assert response.headers[\"Location\"].endswith(\"?_sort_desc=sortable\")\n\n\n@pytest.mark.asyncio\nasync def test_sort_links(ds_client):\n    response = await ds_client.get(\"/fixtures/sortable?_sort=sortable\")\n    assert response.status_code == 200\n    ths = Soup(response.text, \"html.parser\").find_all(\"th\")\n    attrs_and_link_attrs = [\n        {\n            \"attrs\": th.attrs,\n            \"a_href\": (th.find(\"a\")[\"href\"] if th.find(\"a\") else None),\n        }\n        for th in ths\n    ]\n    assert attrs_and_link_attrs == [\n        {\n            \"attrs\": {\n                \"class\": [\"col-Link\"],\n                \"scope\": \"col\",\n                \"data-column\": \"Link\",\n                \"data-column-type\": \"\",\n                \"data-column-not-null\": \"0\",\n                \"data-is-pk\": \"0\",\n                \"data-is-link-column\": \"1\",\n            },\n            \"a_href\": None,\n        },\n        {\n            \"attrs\": {\n                \"class\": [\"col-pk1\"],\n                \"scope\": \"col\",\n                \"data-column\": \"pk1\",\n                \"data-column-type\": \"varchar(30)\",\n                \"data-column-not-null\": \"0\",\n                \"data-is-pk\": \"1\",\n            },\n            \"a_href\": None,\n        },\n        {\n            \"attrs\": {\n                \"class\": [\"col-pk2\"],\n                \"scope\": \"col\",\n                \"data-column\": \"pk2\",\n                \"data-column-type\": \"varchar(30)\",\n                \"data-column-not-null\": \"0\",\n                \"data-is-pk\": \"1\",\n            },\n            \"a_href\": None,\n        },\n        {\n            \"attrs\": {\n                \"class\": [\"col-content\"],\n                \"scope\": \"col\",\n                \"data-column\": \"content\",\n                \"data-column-type\": \"text\",\n                \"data-column-not-null\": \"0\",\n                \"data-is-pk\": \"0\",\n            },\n            \"a_href\": None,\n        },\n        {\n            \"attrs\": {\n                \"class\": [\"col-sortable\"],\n                \"scope\": \"col\",\n                \"data-column\": \"sortable\",\n                \"data-column-type\": \"integer\",\n                \"data-column-not-null\": \"0\",\n                \"data-is-pk\": \"0\",\n            },\n            \"a_href\": \"/fixtures/sortable?_sort_desc=sortable\",\n        },\n        {\n            \"attrs\": {\n                \"class\": [\"col-sortable_with_nulls\"],\n                \"scope\": \"col\",\n                \"data-column\": \"sortable_with_nulls\",\n                \"data-column-type\": \"real\",\n                \"data-column-not-null\": \"0\",\n                \"data-is-pk\": \"0\",\n            },\n            \"a_href\": \"/fixtures/sortable?_sort=sortable_with_nulls\",\n        },\n        {\n            \"attrs\": {\n                \"class\": [\"col-sortable_with_nulls_2\"],\n                \"scope\": \"col\",\n                \"data-column\": \"sortable_with_nulls_2\",\n                \"data-column-type\": \"real\",\n                \"data-column-not-null\": \"0\",\n                \"data-is-pk\": \"0\",\n            },\n            \"a_href\": \"/fixtures/sortable?_sort=sortable_with_nulls_2\",\n        },\n        {\n            \"attrs\": {\n                \"class\": [\"col-text\"],\n                \"scope\": \"col\",\n                \"data-column\": \"text\",\n                \"data-column-type\": \"text\",\n                \"data-column-not-null\": \"0\",\n                \"data-is-pk\": \"0\",\n            },\n            \"a_href\": \"/fixtures/sortable?_sort=text\",\n        },\n    ]\n\n\n@pytest.mark.asyncio\nasync def test_facet_display(ds_client):\n    response = await ds_client.get(\n        \"/fixtures/facetable?_facet=planet_int&_facet=_city_id&_facet=on_earth\"\n    )\n    assert response.status_code == 200\n    soup = Soup(response.text, \"html.parser\")\n    divs = soup.find(\"div\", {\"class\": \"facet-results\"}).find_all(\"div\")\n    actual = []\n    for div in divs:\n        actual.append(\n            {\n                \"name\": div.find(\"strong\").text.split()[0],\n                \"items\": [\n                    {\n                        \"name\": a.text,\n                        \"qs\": a[\"href\"].split(\"?\")[-1],\n                        \"count\": int(str(a.parent).split(\"\")[1].split(\"<\")[0]),\n                    }\n                    for a in div.find(\"ul\").find_all(\"a\")\n                ],\n            }\n        )\n    assert actual == [\n        {\n            \"name\": \"_city_id\",\n            \"items\": [\n                {\n                    \"name\": \"San Francisco\",\n                    \"qs\": \"_facet=planet_int&_facet=_city_id&_facet=on_earth&_city_id__exact=1\",\n                    \"count\": 6,\n                },\n                {\n                    \"name\": \"Los Angeles\",\n                    \"qs\": \"_facet=planet_int&_facet=_city_id&_facet=on_earth&_city_id__exact=2\",\n                    \"count\": 4,\n                },\n                {\n                    \"name\": \"Detroit\",\n                    \"qs\": \"_facet=planet_int&_facet=_city_id&_facet=on_earth&_city_id__exact=3\",\n                    \"count\": 4,\n                },\n                {\n                    \"name\": \"Memnonia\",\n                    \"qs\": \"_facet=planet_int&_facet=_city_id&_facet=on_earth&_city_id__exact=4\",\n                    \"count\": 1,\n                },\n            ],\n        },\n        {\n            \"name\": \"planet_int\",\n            \"items\": [\n                {\n                    \"name\": \"1\",\n                    \"qs\": \"_facet=planet_int&_facet=_city_id&_facet=on_earth&planet_int=1\",\n                    \"count\": 14,\n                },\n                {\n                    \"name\": \"2\",\n                    \"qs\": \"_facet=planet_int&_facet=_city_id&_facet=on_earth&planet_int=2\",\n                    \"count\": 1,\n                },\n            ],\n        },\n        {\n            \"name\": \"on_earth\",\n            \"items\": [\n                {\n                    \"name\": \"1\",\n                    \"qs\": \"_facet=planet_int&_facet=_city_id&_facet=on_earth&on_earth=1\",\n                    \"count\": 14,\n                },\n                {\n                    \"name\": \"0\",\n                    \"qs\": \"_facet=planet_int&_facet=_city_id&_facet=on_earth&on_earth=0\",\n                    \"count\": 1,\n                },\n            ],\n        },\n    ]\n\n\n@pytest.mark.asyncio\nasync def test_facets_persist_through_filter_form(ds_client):\n    response = await ds_client.get(\n        \"/fixtures/facetable?_facet=planet_int&_facet=_city_id&_facet_array=tags\"\n    )\n    assert response.status_code == 200\n    inputs = Soup(response.text, \"html.parser\").find(\"form\").find_all(\"input\")\n    hiddens = [i for i in inputs if i[\"type\"] == \"hidden\"]\n    assert [(hidden[\"name\"], hidden[\"value\"]) for hidden in hiddens] == [\n        (\"_facet\", \"planet_int\"),\n        (\"_facet\", \"_city_id\"),\n        (\"_facet_array\", \"tags\"),\n    ]\n\n\n@pytest.mark.asyncio\nasync def test_next_does_not_persist_in_hidden_field(ds_client):\n    response = await ds_client.get(\"/fixtures/searchable?_size=1&_next=1\")\n    assert response.status_code == 200\n    inputs = Soup(response.text, \"html.parser\").find(\"form\").find_all(\"input\")\n    hiddens = [i for i in inputs if i[\"type\"] == \"hidden\"]\n    assert [(hidden[\"name\"], hidden[\"value\"]) for hidden in hiddens] == [\n        (\"_size\", \"1\"),\n    ]\n\n\n@pytest.mark.asyncio\nasync def test_table_html_simple_primary_key(ds_client):\n    response = await ds_client.get(\"/fixtures/simple_primary_key?_size=3\")\n    assert response.status_code == 200\n    table = Soup(response.text, \"html.parser\").find(\"table\")\n    assert table[\"class\"] == [\"rows-and-columns\"]\n    ths = table.find_all(\"th\")\n    assert \"id\\xa0▼\" == ths[0].find(\"a\").string.strip()\n    for expected_col, th in zip((\"content\",), ths[1:]):\n        a = th.find(\"a\")\n        assert expected_col == a.string\n        assert a[\"href\"].endswith(f\"/simple_primary_key?_size=3&_sort={expected_col}\")\n        assert [\"nofollow\"] == a[\"rel\"]\n    assert [\n        [\n            '',\n            '',\n        ],\n        [\n            '',\n            '',\n        ],\n        [\n            '',\n            '',\n        ],\n    ] == [[str(td) for td in tr.select(\"td\")] for tr in table.select(\"tbody tr\")]\n\n\n@pytest.mark.asyncio\nasync def test_table_csv_json_export_interface(ds_client):\n    response = await ds_client.get(\"/fixtures/simple_primary_key?id__gt=2\")\n    assert response.status_code == 200\n    # The links at the top of the page\n    links = (\n        Soup(response.text, \"html.parser\")\n        .find(\"p\", {\"class\": \"export-links\"})\n        .find_all(\"a\")\n    )\n    actual = [link[\"href\"] for link in links]\n    expected = [\n        \"/fixtures/simple_primary_key.json?id__gt=2\",\n        \"/fixtures/simple_primary_key.testall?id__gt=2\",\n        \"/fixtures/simple_primary_key.testnone?id__gt=2\",\n        \"/fixtures/simple_primary_key.testresponse?id__gt=2\",\n        \"/fixtures/simple_primary_key.csv?id__gt=2&_size=max\",\n        \"#export\",\n    ]\n    assert expected == actual\n    # And the advanced export box at the bottom:\n    div = Soup(response.text, \"html.parser\").find(\"div\", {\"class\": \"advanced-export\"})\n    json_links = [a[\"href\"] for a in div.find(\"p\").find_all(\"a\")]\n    assert [\n        \"/fixtures/simple_primary_key.json?id__gt=2\",\n        \"/fixtures/simple_primary_key.json?id__gt=2&_shape=array\",\n        \"/fixtures/simple_primary_key.json?id__gt=2&_shape=array&_nl=on\",\n        \"/fixtures/simple_primary_key.json?id__gt=2&_shape=object\",\n    ] == json_links\n    # And the CSV form\n    form = div.find(\"form\")\n    assert form[\"action\"].endswith(\"/simple_primary_key.csv\")\n    inputs = [str(input) for input in form.find_all(\"input\")]\n    assert [\n        '',\n        '',\n        '',\n        '',\n    ] == inputs\n\n\n@pytest.mark.asyncio\nasync def test_csv_json_export_links_include_labels_if_foreign_keys(ds_client):\n    response = await ds_client.get(\"/fixtures/facetable\")\n    assert response.status_code == 200\n    links = (\n        Soup(response.text, \"html.parser\")\n        .find(\"p\", {\"class\": \"export-links\"})\n        .find_all(\"a\")\n    )\n    actual = [link[\"href\"] for link in links]\n    expected = [\n        \"/fixtures/facetable.json?_labels=on\",\n        \"/fixtures/facetable.testall?_labels=on\",\n        \"/fixtures/facetable.testnone?_labels=on\",\n        \"/fixtures/facetable.testresponse?_labels=on\",\n        \"/fixtures/facetable.csv?_labels=on&_size=max\",\n        \"#export\",\n    ]\n    assert expected == actual\n\n\n@pytest.mark.asyncio\nasync def test_table_not_exists(ds_client):\n    assert \"Table not found\" in (await ds_client.get(\"/fixtures/blah\")).text\n\n\n@pytest.mark.asyncio\nasync def test_table_html_no_primary_key(ds_client):\n    response = await ds_client.get(\"/fixtures/no_primary_key\")\n    assert response.status_code == 200\n    table = Soup(response.text, \"html.parser\").find(\"table\")\n    # We have disabled sorting for this table using metadata.json\n    assert [\"content\", \"a\", \"b\", \"c\"] == [\n        th.string.strip() for th in table.select(\"thead th\")[2:]\n    ]\n    expected = [\n        [\n            ''.format(\n                i, i\n            ),\n            f'',\n            f'',\n            f'',\n            f'',\n            f'',\n        ]\n        for i in range(1, 51)\n    ]\n    assert expected == [\n        [str(td) for td in tr.select(\"td\")] for tr in table.select(\"tbody tr\")\n    ]\n\n\n@pytest.mark.asyncio\nasync def test_rowid_sortable_no_primary_key(ds_client):\n    response = await ds_client.get(\"/fixtures/no_primary_key\")\n    assert response.status_code == 200\n    table = Soup(response.text, \"html.parser\").find(\"table\")\n    assert table[\"class\"] == [\"rows-and-columns\"]\n    ths = table.find_all(\"th\")\n    assert \"rowid\\xa0▼\" == ths[1].find(\"a\").string.strip()\n\n\n@pytest.mark.asyncio\nasync def test_table_html_compound_primary_key(ds_client):\n    response = await ds_client.get(\"/fixtures/compound_primary_key\")\n    assert response.status_code == 200\n    table = Soup(response.text, \"html.parser\").find(\"table\")\n    ths = table.find_all(\"th\")\n    assert \"Link\" == ths[0].string.strip()\n    for expected_col, th in zip((\"pk1\", \"pk2\", \"content\"), ths[1:]):\n        a = th.find(\"a\")\n        assert expected_col == a.string\n        assert th[\"class\"] == [f\"col-{expected_col}\"]\n        assert a[\"href\"].endswith(f\"/compound_primary_key?_sort={expected_col}\")\n    expected = [\n        [\n            '',\n            '',\n            '',\n            '',\n        ],\n        [\n            '',\n            '',\n            '',\n            '',\n        ],\n        [\n            '',\n            '',\n            '',\n            '',\n        ],\n    ]\n    assert [\n        [str(td) for td in tr.select(\"td\")] for tr in table.select(\"tbody tr\")\n    ] == expected\n\n\n@pytest.mark.asyncio\nasync def test_table_html_foreign_key_links(ds_client):\n    response = await ds_client.get(\"/fixtures/foreign_key_references\")\n    assert response.status_code == 200\n    table = Soup(response.text, \"html.parser\").find(\"table\")\n    actual = [[str(td) for td in tr.select(\"td\")] for tr in table.select(\"tbody tr\")]\n    assert actual == [\n        [\n            '',\n            '',\n            '',\n            '',\n            '',\n            '',\n        ],\n        [\n            '',\n            '',\n            '',\n            '',\n            '',\n            '',\n        ],\n    ]\n\n\n@pytest.mark.asyncio\nasync def test_table_html_foreign_key_facets(ds_client):\n    response = await ds_client.get(\n        \"/fixtures/foreign_key_references?_facet=foreign_key_with_blank_label\"\n    )\n    assert response.status_code == 200\n    assert (\n        '
  • - 1
    • '\n    ) in response.text\n\n\n@pytest.mark.asyncio\nasync def test_table_html_disable_foreign_key_links_with_labels(ds_client):\n    response = await ds_client.get(\n        \"/fixtures/foreign_key_references?_labels=off&_size=1\"\n    )\n    assert response.status_code == 200\n    table = Soup(response.text, \"html.parser\").find(\"table\")\n    actual = [[str(td) for td in tr.select(\"td\")] for tr in table.select(\"tbody tr\")]\n    assert actual == [\n        [\n            '    ',\n            '',\n            '',\n            '',\n            '',\n            '',\n        ]\n    ]\n\n\n@pytest.mark.asyncio\nasync def test_table_html_foreign_key_custom_label_column(ds_client):\n    response = await ds_client.get(\"/fixtures/custom_foreign_key_label\")\n    assert response.status_code == 200\n    table = Soup(response.text, \"html.parser\").find(\"table\")\n    expected = [\n        [\n            '',\n            '',\n        ]\n    ]\n    assert expected == [\n        [str(td) for td in tr.select(\"td\")] for tr in table.select(\"tbody tr\")\n    ]\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"path,expected_column_options\",\n    [\n        (\"/fixtures/infinity\", [\"- column -\", \"rowid\", \"value\"]),\n        (\n            \"/fixtures/primary_key_multiple_columns\",\n            [\"- column -\", \"id\", \"content\", \"content2\"],\n        ),\n        (\"/fixtures/compound_primary_key\", [\"- column -\", \"pk1\", \"pk2\", \"content\"]),\n    ],\n)\nasync def test_table_html_filter_form_column_options(\n    path, expected_column_options, ds_client\n):\n    response = await ds_client.get(path)\n    assert response.status_code == 200\n    form = Soup(response.text, \"html.parser\").find(\"form\")\n    column_options = [\n        o.attrs.get(\"value\") or o.string\n        for o in form.select(\"select[name=_filter_column] option\")\n    ]\n    assert expected_column_options == column_options\n\n\n@pytest.mark.asyncio\nasync def test_table_html_filter_form_still_shows_nocol_columns(ds_client):\n    # https://github.com/simonw/datasette/issues/1503\n    response = await ds_client.get(\"/fixtures/sortable?_nocol=sortable\")\n    assert response.status_code == 200\n    form = Soup(response.text, \"html.parser\").find(\"form\")\n    assert [\n        o.string\n        for o in form.select(\"select[name='_filter_column']\")[0].select(\"option\")\n    ] == [\n        \"- column -\",\n        \"pk1\",\n        \"pk2\",\n        \"content\",\n        \"sortable_with_nulls\",\n        \"sortable_with_nulls_2\",\n        \"text\",\n        # Moved to the end because it is no longer returned by the query:\n        \"sortable\",\n    ]\n\n\n@pytest.mark.asyncio\nasync def test_column_chooser_present(ds_client):\n    response = await ds_client.get(\"/fixtures/facetable\")\n    assert response.status_code == 200\n    soup = Soup(response.text, \"html.parser\")\n    # Web component should be present\n    chooser = soup.find(\"column-chooser\")\n    assert chooser is not None\n    # Script block should contain column data as JSON\n\n    scripts = soup.find_all(\"script\")\n    chooser_script = [s for s in scripts if \"_columnChooserData\" in (s.string or \"\")]\n    assert len(chooser_script) == 1\n    script_text = chooser_script[0].string\n    # Extract the JSON data\n    assert \"allColumns\" in script_text\n    assert \"selectedColumns\" in script_text\n    assert \"primaryKeys\" in script_text\n\n\n@pytest.mark.asyncio\nasync def test_mobile_column_actions_present(ds_client):\n    response = await ds_client.get(\"/fixtures/facetable\")\n    assert response.status_code == 200\n    soup = Soup(response.text, \"html.parser\")\n    button = soup.select_one(\"button.column-actions-mobile.small-screen-only\")\n    assert button is not None\n    assert button.text.strip() == \"Column actions\"\n    assert button.find(\"svg\") is not None\n    assert any(\n        \"mobile-column-actions.js\" in (script.get(\"src\") or \"\")\n        for script in soup.find_all(\"script\")\n    )\n\n\n@pytest.mark.asyncio\nasync def test_column_chooser_data_reflects_col_filtering(ds_client):\n    response = await ds_client.get(\"/fixtures/facetable?_col=state&_col=created\")\n    assert response.status_code == 200\n    import json\n    import re\n\n    soup = Soup(response.text, \"html.parser\")\n    chooser = soup.find(\"column-chooser\")\n    assert chooser is not None\n    scripts = soup.find_all(\"script\")\n    chooser_script = [s for s in scripts if \"_columnChooserData\" in (s.string or \"\")]\n    script_text = chooser_script[0].string\n    # Parse the JSON object from the script\n    match = re.search(\n        r\"window\\._columnChooserData\\s*=\\s*({.*?});\", script_text, re.DOTALL\n    )\n    data = json.loads(match.group(1))\n    # All non-PK columns should still be listed in allColumns\n    assert \"state\" in data[\"allColumns\"]\n    assert \"created\" in data[\"allColumns\"]\n    assert \"planet_int\" in data[\"allColumns\"]\n    # Only state and created should be in selectedColumns (plus pk)\n    non_pk_selected = [\n        c for c in data[\"selectedColumns\"] if c not in data[\"primaryKeys\"]\n    ]\n    assert \"state\" in non_pk_selected\n    assert \"created\" in non_pk_selected\n    assert \"planet_int\" not in non_pk_selected\n\n\n@pytest.mark.asyncio\nasync def test_column_chooser_shown_for_views(ds_client):\n    response = await ds_client.get(\"/fixtures/simple_view\")\n    assert response.status_code == 200\n    soup = Soup(response.text, \"html.parser\")\n    chooser = soup.find(\"column-chooser\")\n    assert chooser is not None\n    scripts = soup.find_all(\"script\")\n    chooser_script = [s for s in scripts if \"_columnChooserData\" in (s.string or \"\")]\n    assert len(chooser_script) == 1\n\n\n@pytest.mark.asyncio\nasync def test_compound_primary_key_with_foreign_key_references(ds_client):\n    # e.g. a many-to-many table with a compound primary key on the two columns\n    response = await ds_client.get(\"/fixtures/searchable_tags\")\n    assert response.status_code == 200\n    table = Soup(response.text, \"html.parser\").find(\"table\")\n    expected = [\n        [\n            '',\n            '',\n            '',\n        ],\n        [\n            '',\n            '',\n            '',\n        ],\n    ]\n    assert expected == [\n        [str(td) for td in tr.select(\"td\")] for tr in table.select(\"tbody tr\")\n    ]\n\n\n@pytest.mark.asyncio\nasync def test_view_html(ds_client):\n    response = await ds_client.get(\"/fixtures/simple_view?_size=3\")\n    assert response.status_code == 200\n    table = Soup(response.text, \"html.parser\").find(\"table\")\n    ths = table.select(\"thead th\")\n    assert 2 == len(ths)\n    assert ths[0].find(\"a\") is not None\n    assert ths[0].find(\"a\")[\"href\"].endswith(\"/simple_view?_size=3&_sort=content\")\n    assert ths[0].find(\"a\").string.strip() == \"content\"\n    assert ths[1].find(\"a\") is None\n    assert ths[1].string.strip() == \"upper_content\"\n    expected = [\n        [\n            '',\n            '',\n        ],\n        [\n            '',\n            '',\n        ],\n        [\n            '',\n            '',\n        ],\n    ]\n    assert expected == [\n        [str(td) for td in tr.select(\"td\")] for tr in table.select(\"tbody tr\")\n    ]\n\n\n@pytest.mark.asyncio\nasync def test_table_metadata(ds_client):\n    response = await ds_client.get(\"/fixtures/simple_primary_key\")\n    assert response.status_code == 200\n    soup = Soup(response.text, \"html.parser\")\n    # Page title should be custom and should be HTML escaped\n    assert \"This <em>HTML</em> is escaped\" == inner_html(soup.find(\"h1\"))\n    # Description should be custom and NOT escaped (we used description_html)\n    assert \"Simple primary key\" == inner_html(\n        soup.find(\"div\", {\"class\": \"metadata-description\"})\n    )\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"path,has_object,has_stream,has_expand\",\n    [\n        (\"/fixtures/no_primary_key\", False, True, False),\n        (\"/fixtures/complex_foreign_keys\", True, False, True),\n    ],\n)\nasync def test_advanced_export_box(ds_client, path, has_object, has_stream, has_expand):\n    response = await ds_client.get(path)\n    assert response.status_code == 200\n    soup = Soup(response.text, \"html.parser\")\n    # JSON shape options\n    expected_json_shapes = [\"default\", \"array\", \"newline-delimited\"]\n    if has_object:\n        expected_json_shapes.append(\"object\")\n    div = soup.find(\"div\", {\"class\": \"advanced-export\"})\n    assert expected_json_shapes == [a.text for a in div.find(\"p\").find_all(\"a\")]\n    # \"stream all rows\" option\n    if has_stream:\n        assert \"stream all rows\" in str(div)\n    # \"expand labels\" option\n    if has_expand:\n        assert \"expand labels\" in str(div)\n\n\n@pytest.mark.asyncio\nasync def test_extra_where_clauses(ds_client):\n    response = await ds_client.get(\n        \"/fixtures/facetable?_where=_neighborhood='Dogpatch'&_where=_city_id=1\"\n    )\n    soup = Soup(response.text, \"html.parser\")\n    div = soup.select(\".extra-wheres\")[0]\n    assert \"2 extra where clauses\" == div.find(\"h3\").text\n    hrefs = [a[\"href\"] for a in div.find_all(\"a\")]\n    assert [\n        \"/fixtures/facetable?_where=_city_id%3D1\",\n        \"/fixtures/facetable?_where=_neighborhood%3D%27Dogpatch%27\",\n    ] == hrefs\n    # These should also be persisted as hidden fields\n    inputs = soup.find(\"form\").find_all(\"input\")\n    hiddens = [i for i in inputs if i[\"type\"] == \"hidden\"]\n    assert [(\"_where\", \"_neighborhood='Dogpatch'\"), (\"_where\", \"_city_id=1\")] == [\n        (hidden[\"name\"], hidden[\"value\"]) for hidden in hiddens\n    ]\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"path,expected_hidden\",\n    [\n        (\"/fixtures/facetable?_size=10\", [(\"_size\", \"10\")]),\n        (\n            \"/fixtures/facetable?_size=10&_ignore=1&_ignore=2\",\n            [\n                (\"_size\", \"10\"),\n                (\"_ignore\", \"1\"),\n                (\"_ignore\", \"2\"),\n            ],\n        ),\n    ],\n)\nasync def test_other_hidden_form_fields(ds_client, path, expected_hidden):\n    response = await ds_client.get(path)\n    soup = Soup(response.text, \"html.parser\")\n    inputs = soup.find(\"form\").find_all(\"input\")\n    hiddens = [i for i in inputs if i[\"type\"] == \"hidden\"]\n    assert [(hidden[\"name\"], hidden[\"value\"]) for hidden in hiddens] == expected_hidden\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"path,expected_hidden\",\n    [\n        (\"/fixtures/searchable?_search=terry\", []),\n        (\"/fixtures/searchable?_sort=text2\", []),\n        (\"/fixtures/searchable?_sort_desc=text2\", []),\n        (\"/fixtures/searchable?_sort=text2&_where=1\", [(\"_where\", \"1\")]),\n    ],\n)\nasync def test_search_and_sort_fields_not_duplicated(ds_client, path, expected_hidden):\n    # https://github.com/simonw/datasette/issues/1214\n    response = await ds_client.get(path)\n    soup = Soup(response.text, \"html.parser\")\n    inputs = soup.find(\"form\").find_all(\"input\")\n    hiddens = [i for i in inputs if i[\"type\"] == \"hidden\"]\n    assert [(hidden[\"name\"], hidden[\"value\"]) for hidden in hiddens] == expected_hidden\n\n\n@pytest.mark.asyncio\nasync def test_binary_data_display_in_table(ds_client):\n    response = await ds_client.get(\"/fixtures/binary_data\")\n    assert response.status_code == 200\n    table = Soup(response.text, \"html.parser\").find(\"table\")\n    expected_tds = [\n        [\n            '',\n            '',\n            '',\n        ],\n        [\n            '',\n            '',\n            '',\n        ],\n        [\n            '',\n            '',\n            '',\n        ],\n    ]\n    assert expected_tds == [\n        [str(td) for td in tr.select(\"td\")] for tr in table.select(\"tbody tr\")\n    ]\n\n\ndef test_custom_table_include():\n    with make_app_client(\n        template_dir=str(pathlib.Path(__file__).parent / \"test_templates\")\n    ) as client:\n        response = client.get(\"/fixtures/complex_foreign_keys\")\n        assert response.status == 200\n        assert (\n            '
    '\n            '1 - 2 - hello 1'\n            \"
    \"\n        ) == str(Soup(response.text, \"html.parser\").select_one(\"div.custom-table-row\"))\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\"json\", (True, False))\n@pytest.mark.parametrize(\n    \"params,error\",\n    (\n        (\"?_sort=bad\", \"Cannot sort table by bad\"),\n        (\"?_sort_desc=bad\", \"Cannot sort table by bad\"),\n        (\n            \"?_sort=state&_sort_desc=state\",\n            \"Cannot use _sort and _sort_desc at the same time\",\n        ),\n    ),\n)\nasync def test_sort_errors(ds_client, json, params, error):\n    path = \"/fixtures/facetable{}{}\".format(\n        \".json\" if json else \"\",\n        params,\n    )\n    response = await ds_client.get(path)\n    assert response.status_code == 400\n    if json:\n        assert response.json() == {\n            \"ok\": False,\n            \"error\": error,\n            \"status\": 400,\n            \"title\": None,\n        }\n    else:\n        assert error in response.text\n\n\n@pytest.mark.asyncio\nasync def test_metadata_sort(ds_client):\n    response = await ds_client.get(\"/fixtures/facet_cities\")\n    assert response.status_code == 200\n    table = Soup(response.text, \"html.parser\").find(\"table\")\n    assert table[\"class\"] == [\"rows-and-columns\"]\n    ths = table.find_all(\"th\")\n    assert [\"id\", \"name\\xa0▼\"] == [th.find(\"a\").string.strip() for th in ths]\n    rows = [[str(td) for td in tr.select(\"td\")] for tr in table.select(\"tbody tr\")]\n    expected = [\n        [\n            '    ',\n            '',\n        ],\n        [\n            '',\n            '',\n        ],\n        [\n            '',\n            '',\n        ],\n        [\n            '',\n            '',\n        ],\n    ]\n    assert expected == rows\n    # Make sure you can reverse that sort order\n    response = await ds_client.get(\"/fixtures/facet_cities?_sort_desc=name\")\n    assert response.status_code == 200\n    table = Soup(response.text, \"html.parser\").find(\"table\")\n    rows = [[str(td) for td in tr.select(\"td\")] for tr in table.select(\"tbody tr\")]\n    assert list(reversed(expected)) == rows\n\n\n@pytest.mark.asyncio\nasync def test_metadata_sort_desc(ds_client):\n    response = await ds_client.get(\"/fixtures/attraction_characteristic\")\n    assert response.status_code == 200\n    table = Soup(response.text, \"html.parser\").find(\"table\")\n    assert table[\"class\"] == [\"rows-and-columns\"]\n    ths = table.find_all(\"th\")\n    assert [\"pk\\xa0▲\", \"name\"] == [th.find(\"a\").string.strip() for th in ths]\n    rows = [[str(td) for td in tr.select(\"td\")] for tr in table.select(\"tbody tr\")]\n    expected = [\n        [\n            '',\n            '',\n        ],\n        [\n            '',\n            '',\n        ],\n    ]\n    assert expected == rows\n    # Make sure you can reverse that sort order\n    response = await ds_client.get(\"/fixtures/attraction_characteristic?_sort=pk\")\n    assert response.status_code == 200\n    table = Soup(response.text, \"html.parser\").find(\"table\")\n    rows = [[str(td) for td in tr.select(\"td\")] for tr in table.select(\"tbody tr\")]\n    assert list(reversed(expected)) == rows\n\n\n@pytest.mark.parametrize(\n    \"max_returned_rows,path,expected_num_facets,expected_ellipses,expected_ellipses_url\",\n    (\n        (\n            5,\n            # Default should show 2 facets\n            \"/fixtures/facetable?_facet=_neighborhood\",\n            2,\n            True,\n            \"/fixtures/facetable?_facet=_neighborhood&_facet_size=max\",\n        ),\n        # _facet_size above max_returned_rows should show max_returned_rows (5)\n        (\n            5,\n            \"/fixtures/facetable?_facet=_neighborhood&_facet_size=50\",\n            5,\n            True,\n            \"/fixtures/facetable?_facet=_neighborhood&_facet_size=max\",\n        ),\n        # If max_returned_rows is high enough, should return all\n        (\n            20,\n            \"/fixtures/facetable?_facet=_neighborhood&_facet_size=max\",\n            14,\n            False,\n            None,\n        ),\n        # If num facets > max_returned_rows, show ... without a link\n        # _facet_size above max_returned_rows should show max_returned_rows (5)\n        (\n            5,\n            \"/fixtures/facetable?_facet=_neighborhood&_facet_size=max\",\n            5,\n            True,\n            None,\n        ),\n    ),\n)\ndef test_facet_more_links(\n    max_returned_rows,\n    path,\n    expected_num_facets,\n    expected_ellipses,\n    expected_ellipses_url,\n):\n    with make_app_client(\n        settings={\"max_returned_rows\": max_returned_rows, \"default_facet_size\": 2}\n    ) as client:\n        response = client.get(path)\n        soup = Soup(response.body, \"html.parser\")\n        lis = soup.select(\"#facet-neighborhood-b352a7 ul li:not(.facet-truncated)\")\n        facet_truncated = soup.select_one(\".facet-truncated\")\n        assert len(lis) == expected_num_facets\n        if not expected_ellipses:\n            assert facet_truncated is None\n        else:\n            if expected_ellipses_url:\n                assert facet_truncated.find(\"a\")[\"href\"] == expected_ellipses_url\n            else:\n                assert facet_truncated.find(\"a\") is None\n\n\ndef test_unavailable_table_does_not_break_sort_relationships():\n    # https://github.com/simonw/datasette/issues/1305\n    with make_app_client(\n        config={\n            \"databases\": {\n                \"fixtures\": {\"tables\": {\"foreign_key_references\": {\"allow\": False}}}\n            }\n        }\n    ) as client:\n        response = client.get(\"/?_sort=relationships\")\n        assert response.status == 200\n\n\n@pytest.mark.asyncio\nasync def test_column_metadata(ds_client):\n    response = await ds_client.get(\"/fixtures/roadside_attractions\")\n    soup = Soup(response.text, \"html.parser\")\n    dl = soup.find(\"dl\")\n    assert [(dt.text, dt.next_sibling.text) for dt in dl.find_all(\"dt\")] == [\n        (\"address\", \"The street address for the attraction\"),\n        (\"name\", \"The name of the attraction\"),\n    ]\n    assert (\n        soup.select(\"th[data-column=name]\")[0][\"data-column-description\"]\n        == \"The name of the attraction\"\n    )\n    assert (\n        soup.select(\"th[data-column=address]\")[0][\"data-column-description\"]\n        == \"The street address for the attraction\"\n    )\n\n\ndef test_facet_total():\n    # https://github.com/simonw/datasette/issues/1423\n    # https://github.com/simonw/datasette/issues/1556\n    with make_app_client(settings={\"max_returned_rows\": 100}) as client:\n        path = \"/fixtures/sortable?_facet=content&_facet=pk1\"\n        response = client.get(path)\n        assert response.status == 200\n    fragments = (\n        '>30',\n        '8',\n    )\n    for fragment in fragments:\n        assert fragment in response.text\n\n\n@pytest.mark.asyncio\nasync def test_sort_rowid_with_next(ds_client):\n    # https://github.com/simonw/datasette/issues/1470\n    response = await ds_client.get(\"/fixtures/binary_data?_size=1&_next=1&_sort=rowid\")\n    assert response.status_code == 200\n\n\ndef assert_querystring_equal(expected, actual):\n    assert sorted(expected.split(\"&\")) == sorted(actual.split(\"&\"))\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"path,expected\",\n    (\n        (\n            \"/fixtures/facetable\",\n            \"fixtures: facetable: 15 rows\",\n        ),\n        (\n            \"/fixtures/facetable?on_earth__exact=1\",\n            \"fixtures: facetable: 14 rows where on_earth = 1\",\n        ),\n    ),\n)\nasync def test_table_page_title(ds_client, path, expected):\n    response = await ds_client.get(path)\n    title = Soup(response.text, \"html.parser\").find(\"title\").text\n    assert title == expected\n\n\n@pytest.mark.asyncio\nasync def test_table_post_method_not_allowed(ds_client):\n    response = await ds_client.post(\"/fixtures/facetable\")\n    assert response.status_code == 405\n    assert \"Method not allowed\" in response.text\n\n\n@pytest.mark.parametrize(\"allow_facet\", (True, False))\ndef test_allow_facet_off(allow_facet):\n    with make_app_client(settings={\"allow_facet\": allow_facet}) as client:\n        response = client.get(\"/fixtures/facetable\")\n        expected = \"DATASETTE_ALLOW_FACET = {};\".format(\n            \"true\" if allow_facet else \"false\"\n        )\n        assert expected in response.text\n        if allow_facet:\n            assert \"Suggested facets\" in response.text\n        else:\n            assert \"Suggested facets\" not in response.text\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"size,title,length_bytes\",\n    (\n        (2000, ' title=\"2.0 KB\"', \"2,000\"),\n        (20000, ' title=\"19.5 KB\"', \"20,000\"),\n        (20, \"\", \"20\"),\n    ),\n)\nasync def test_format_of_binary_links(size, title, length_bytes):\n    ds = Datasette()\n    db_name = \"binary-links-{}\".format(size)\n    db = ds.add_memory_database(db_name)\n    sql = \"select zeroblob({}) as blob\".format(size)\n    await db.execute_write(\"create table blobs as {}\".format(sql))\n    response = await ds.client.get(\"/{}/blobs\".format(db_name))\n    assert response.status_code == 200\n    expected = \"{}><Binary: {} bytes>\".format(title, length_bytes)\n    assert expected in response.text\n    # And test with arbitrary SQL query too\n    sql_response = await ds.client.get(\n        \"{}/-/query\".format(db_name), params={\"sql\": sql}\n    )\n    assert sql_response.status_code == 200\n    assert expected in sql_response.text\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"config\",\n    (\n        # Blocked at table level\n        {\n            \"databases\": {\n                \"foreign_key_labels\": {\n                    \"tables\": {\n                        # Table a is only visible to root\n                        \"a\": {\"allow\": {\"id\": \"root\"}},\n                    }\n                }\n            }\n        },\n        # Blocked at database level\n        {\n            \"databases\": {\n                \"foreign_key_labels\": {\n                    # Only root can view this database\n                    \"allow\": {\"id\": \"root\"},\n                    \"tables\": {\n                        # But table b is visible to everyone\n                        \"b\": {\"allow\": True},\n                    },\n                }\n            }\n        },\n        # Blocked at the instance level\n        {\n            \"allow\": {\"id\": \"root\"},\n            \"databases\": {\n                \"foreign_key_labels\": {\n                    \"tables\": {\n                        # Table b is visible to everyone\n                        \"b\": {\"allow\": True},\n                    }\n                }\n            },\n        },\n    ),\n)\nasync def test_foreign_key_labels_obey_permissions(config):\n    ds = Datasette(config=config)\n    db = ds.add_memory_database(\"foreign_key_labels\")\n    await db.execute_write(\n        \"create table if not exists a(id integer primary key, name text)\"\n    )\n    await db.execute_write(\"insert or replace into a (id, name) values (1, 'hello')\")\n    await db.execute_write(\n        \"create table if not exists b(id integer primary key, name text, a_id integer references a(id))\"\n    )\n    await db.execute_write(\n        \"insert or replace into b (id, name, a_id) values (1, 'world', 1)\"\n    )\n    # Anonymous user can see table b but not table a\n    await ds.client.get(\"/foreign_key_labels.json\")\n    anon_a = await ds.client.get(\"/foreign_key_labels/a.json?_labels=on\")\n    assert anon_a.status_code == 403\n    anon_b = await ds.client.get(\"/foreign_key_labels/b.json?_labels=on\")\n    assert anon_b.status_code == 200\n    # root user can see both\n    cookies = {\"ds_actor\": ds.sign({\"a\": {\"id\": \"root\"}}, \"actor\")}\n    root_a = await ds.client.get(\n        \"/foreign_key_labels/a.json?_labels=on\", cookies=cookies\n    )\n    assert root_a.status_code == 200\n    root_b = await ds.client.get(\n        \"/foreign_key_labels/b.json?_labels=on\", cookies=cookies\n    )\n    assert root_b.status_code == 200\n    # Labels should have been expanded for root\n    assert root_b.json() == {\n        \"ok\": True,\n        \"next\": None,\n        \"rows\": [{\"id\": 1, \"name\": \"world\", \"a_id\": {\"value\": 1, \"label\": \"hello\"}}],\n        \"truncated\": False,\n    }\n    # But not for anon\n    assert anon_b.json() == {\n        \"ok\": True,\n        \"next\": None,\n        \"rows\": [{\"id\": 1, \"name\": \"world\", \"a_id\": 1}],\n        \"truncated\": False,\n    }\n\n\ndef test_foreign_keys_special_character_in_database_name(app_client_with_dot):\n    # https://github.com/simonw/datasette/pull/2476\n    response = app_client_with_dot.get(\"/fixtures~2Edot/complex_foreign_keys\")\n    assert 'world' in response.text\n"
  },
  {
    "path": "tests/test_templates/_table.html",
    "content": "{% for row in display_rows %}\n    
    {{ row[\"f1\"] }} - {{ row[\"f2\"] }} - {{ row.display(\"f3\") }}
    \n{% endfor %}\n"
  },
  {
    "path": "tests/test_templates/pages/202.html",
    "content": "{{ custom_status(202) }}202!"
  },
  {
    "path": "tests/test_templates/pages/about.html",
    "content": "ABOUT! view_name:{{ view_name }}"
  },
  {
    "path": "tests/test_templates/pages/atom.html",
    "content": "{{ custom_header(\"content-type\", \"application/xml\") }}"
  },
  {
    "path": "tests/test_templates/pages/headers.html",
    "content": "{{ custom_header(\"x-this-is-foo\", \"foo\") }}FOO{{ custom_header(\"x-this-is-bar\", \"bar\") }}BAR"
  },
  {
    "path": "tests/test_templates/pages/nested/nest.html",
    "content": "Nest!"
  },
  {
    "path": "tests/test_templates/pages/redirect.html",
    "content": "{{ custom_redirect(\"/example\") }}"
  },
  {
    "path": "tests/test_templates/pages/redirect2.html",
    "content": "{{ custom_redirect(\"/example\", 301) }}"
  },
  {
    "path": "tests/test_templates/pages/request.html",
    "content": "path:{{ request.path }}"
  },
  {
    "path": "tests/test_templates/pages/route_{name}.html",
    "content": "{% if name == \"OhNo\" %}{{ raise_404(\"Oh no\") }}{% endif %}\n
    Hello from {{ name }}
    "
  },
  {
    "path": "tests/test_templates/pages/topic_{topic}/{slug}.html",
    "content": "Slug: {{ slug }}, Topic: {{ topic }}"
  },
  {
    "path": "tests/test_templates/pages/topic_{topic}.html",
    "content": "Topic page for {{ topic }}"
  },
  {
    "path": "tests/test_templates/show_json.html",
    "content": "{% extends \"base.html\" %}\n\n{% block content %}\n{{ super() }}\nTest data for extra_template_vars:\n
    {{ extra_template_vars|safe }}
    \n
    {{ extra_template_vars_from_awaitable|safe }}
    \n
    {{ query_database(\"select sqlite_version();\") }}
    \n{% endblock %}\n"
  },
  {
    "path": "tests/test_token_handler.py",
    "content": "\"\"\"\nTests for the register_token_handler plugin hook.\n\"\"\"\n\nfrom datasette.app import Datasette\nfrom datasette.hookspecs import hookimpl\nfrom datasette.plugins import pm\nfrom datasette.tokens import TokenHandler, TokenRestrictions, SignedTokenHandler\nimport pytest\n\n\n@pytest.fixture\ndef datasette():\n    return Datasette()\n\n\n@pytest.mark.asyncio\nasync def test_default_signed_handler_registered(datasette):\n    \"\"\"The default SignedTokenHandler should be registered automatically.\"\"\"\n    handlers = datasette._token_handlers()\n    assert len(handlers) >= 1\n    assert any(isinstance(h, SignedTokenHandler) for h in handlers)\n    assert any(h.name == \"signed\" for h in handlers)\n\n\n@pytest.mark.asyncio\nasync def test_create_token_default(datasette):\n    \"\"\"create_token() with handler='signed' should create a signed token.\"\"\"\n    token = await datasette.create_token(\"test_actor\", handler=\"signed\")\n    assert token.startswith(\"dstok_\")\n\n\n@pytest.mark.asyncio\nasync def test_create_token_with_restrictions(datasette):\n    \"\"\"create_token() should handle restriction parameters.\"\"\"\n    token = await datasette.create_token(\n        \"test_actor\",\n        handler=\"signed\",\n        expires_after=3600,\n        restrictions=TokenRestrictions().allow_all(\"view-instance\"),\n    )\n    assert token.startswith(\"dstok_\")\n    # Verify the token contains the expected data\n    decoded = datasette.unsign(token[len(\"dstok_\") :], namespace=\"token\")\n    assert decoded[\"a\"] == \"test_actor\"\n    assert decoded[\"d\"] == 3600\n    assert \"_r\" in decoded\n    assert \"a\" in decoded[\"_r\"]\n\n\n@pytest.mark.asyncio\nasync def test_verify_token_default(datasette):\n    \"\"\"verify_token() should verify signed tokens.\"\"\"\n    token = await datasette.create_token(\"test_actor\", handler=\"signed\")\n    actor = await datasette.verify_token(token)\n    assert actor is not None\n    assert actor[\"id\"] == \"test_actor\"\n    assert actor[\"token\"] == \"dstok\"\n\n\n@pytest.mark.asyncio\nasync def test_verify_token_unknown_returns_none(datasette):\n    \"\"\"verify_token() should return None for unrecognized tokens.\"\"\"\n    result = await datasette.verify_token(\"unknown_token_format_xyz\")\n    assert result is None\n\n\n@pytest.mark.asyncio\nasync def test_verify_token_bad_signature_returns_none(datasette):\n    \"\"\"verify_token() should return None for tokens with bad signatures.\"\"\"\n    result = await datasette.verify_token(\"dstok_tampered_data_here\")\n    assert result is None\n\n\n@pytest.mark.asyncio\nasync def test_create_token_with_named_handler(datasette):\n    \"\"\"create_token(handler='signed') should select the signed handler.\"\"\"\n    token = await datasette.create_token(\"test_actor\", handler=\"signed\")\n    assert token.startswith(\"dstok_\")\n\n\n@pytest.mark.asyncio\nasync def test_create_token_unknown_handler_raises(datasette):\n    \"\"\"create_token(handler='nonexistent') should raise ValueError.\"\"\"\n    with pytest.raises(ValueError, match=\"Token handler 'nonexistent' not found\"):\n        await datasette.create_token(\"test_actor\", handler=\"nonexistent\")\n\n\n@pytest.mark.asyncio\nasync def test_custom_token_handler(datasette):\n    \"\"\"A custom token handler should be usable for both create and verify.\"\"\"\n\n    class CustomHandler(TokenHandler):\n        name = \"custom\"\n\n        async def create_token(self, datasette, actor_id, **kwargs):\n            return f\"custom_{actor_id}\"\n\n        async def verify_token(self, datasette, token):\n            if token.startswith(\"custom_\"):\n                return {\"id\": token[len(\"custom_\") :], \"token\": \"custom\"}\n            return None\n\n    class Plugin:\n        __name__ = \"CustomTokenPlugin\"\n\n        @staticmethod\n        @hookimpl\n        def register_token_handler(datasette):\n            return CustomHandler()\n\n    pm.register(Plugin(), name=\"test_custom_handler\")\n    try:\n        handlers = datasette._token_handlers()\n        assert any(h.name == \"custom\" for h in handlers)\n\n        # Create with custom handler\n        token = await datasette.create_token(\"alice\", handler=\"custom\")\n        assert token == \"custom_alice\"\n\n        # Verify custom token\n        actor = await datasette.verify_token(\"custom_alice\")\n        assert actor is not None\n        assert actor[\"id\"] == \"alice\"\n        assert actor[\"token\"] == \"custom\"\n\n        # Signed tokens should still work\n        signed_token = await datasette.create_token(\"bob\", handler=\"signed\")\n        assert signed_token.startswith(\"dstok_\")\n        actor = await datasette.verify_token(signed_token)\n        assert actor[\"id\"] == \"bob\"\n    finally:\n        pm.unregister(name=\"test_custom_handler\")\n\n\n@pytest.mark.asyncio\nasync def test_verify_token_tries_all_handlers(datasette):\n    \"\"\"verify_token() should try each handler until one matches.\"\"\"\n\n    class HandlerA(TokenHandler):\n        name = \"handler_a\"\n\n        async def create_token(self, datasette, actor_id, **kwargs):\n            return f\"a_{actor_id}\"\n\n        async def verify_token(self, datasette, token):\n            if token.startswith(\"a_\"):\n                return {\"id\": token[2:], \"token\": \"handler_a\"}\n            return None\n\n    class HandlerB(TokenHandler):\n        name = \"handler_b\"\n\n        async def create_token(self, datasette, actor_id, **kwargs):\n            return f\"b_{actor_id}\"\n\n        async def verify_token(self, datasette, token):\n            if token.startswith(\"b_\"):\n                return {\"id\": token[2:], \"token\": \"handler_b\"}\n            return None\n\n    class PluginA:\n        __name__ = \"PluginA\"\n\n        @staticmethod\n        @hookimpl\n        def register_token_handler(datasette):\n            return HandlerA()\n\n    class PluginB:\n        __name__ = \"PluginB\"\n\n        @staticmethod\n        @hookimpl\n        def register_token_handler(datasette):\n            return HandlerB()\n\n    pm.register(PluginA(), name=\"test_handler_a\")\n    pm.register(PluginB(), name=\"test_handler_b\")\n    try:\n        # Both handler tokens should verify\n        actor_a = await datasette.verify_token(\"a_alice\")\n        assert actor_a is not None\n        assert actor_a[\"id\"] == \"alice\"\n        assert actor_a[\"token\"] == \"handler_a\"\n\n        actor_b = await datasette.verify_token(\"b_bob\")\n        assert actor_b is not None\n        assert actor_b[\"id\"] == \"bob\"\n        assert actor_b[\"token\"] == \"handler_b\"\n\n        # Unknown token should return None\n        assert await datasette.verify_token(\"c_charlie\") is None\n    finally:\n        pm.unregister(name=\"test_handler_a\")\n        pm.unregister(name=\"test_handler_b\")\n\n\n@pytest.mark.asyncio\nasync def test_token_handler_via_http(datasette):\n    \"\"\"Default signed tokens should work through HTTP auth.\"\"\"\n    token = await datasette.create_token(\"http_user\", handler=\"signed\")\n    response = await datasette.client.get(\n        \"/-/actor.json\",\n        headers={\"Authorization\": f\"Bearer {token}\"},\n    )\n    assert response.status_code == 200\n    actor = response.json()[\"actor\"]\n    assert actor[\"id\"] == \"http_user\"\n    assert actor[\"token\"] == \"dstok\"\n\n\n@pytest.mark.asyncio\nasync def test_custom_handler_via_http(datasette):\n    \"\"\"Custom handler tokens should work through HTTP auth.\"\"\"\n\n    class CustomHandler(TokenHandler):\n        name = \"custom_http\"\n\n        async def create_token(self, datasette, actor_id, **kwargs):\n            return f\"chttp_{actor_id}\"\n\n        async def verify_token(self, datasette, token):\n            if token.startswith(\"chttp_\"):\n                return {\"id\": token[len(\"chttp_\") :], \"token\": \"custom_http\"}\n            return None\n\n    class Plugin:\n        __name__ = \"CustomHTTPPlugin\"\n\n        @staticmethod\n        @hookimpl\n        def register_token_handler(datasette):\n            return CustomHandler()\n\n    pm.register(Plugin(), name=\"test_custom_http\")\n    try:\n        token = await datasette.create_token(\"web_user\", handler=\"custom_http\")\n        response = await datasette.client.get(\n            \"/-/actor.json\",\n            headers={\"Authorization\": f\"Bearer {token}\"},\n        )\n        assert response.status_code == 200\n        actor = response.json()[\"actor\"]\n        assert actor[\"id\"] == \"web_user\"\n        assert actor[\"token\"] == \"custom_http\"\n    finally:\n        pm.unregister(name=\"test_custom_http\")\n\n\n@pytest.mark.asyncio\nasync def test_token_handler_base_class_raises():\n    \"\"\"TokenHandler base class methods should raise NotImplementedError.\"\"\"\n    handler = TokenHandler()\n    ds = Datasette()\n    with pytest.raises(NotImplementedError):\n        await handler.create_token(ds, \"test\")\n    with pytest.raises(NotImplementedError):\n        await handler.verify_token(ds, \"test\")\n\n\n@pytest.mark.asyncio\nasync def test_restrictions_round_trip(datasette):\n    \"\"\"Tokens with database/resource restrictions should round-trip correctly.\"\"\"\n    restrictions = (\n        TokenRestrictions()\n        .allow_all(\"view-instance\")\n        .allow_database(\"docs\", \"view-query\")\n        .allow_resource(\"docs\", \"attachments\", \"insert-row\")\n    )\n    token = await datasette.create_token(\n        \"test_actor\", handler=\"signed\", restrictions=restrictions\n    )\n    actor = await datasette.verify_token(token)\n    assert actor is not None\n    assert actor[\"id\"] == \"test_actor\"\n    assert actor[\"_r\"][\"a\"] == [\"view-instance\"]\n    assert actor[\"_r\"][\"d\"] == {\"docs\": [\"view-query\"]}\n    assert actor[\"_r\"][\"r\"] == {\"docs\": {\"attachments\": [\"insert-row\"]}}\n\n\n@pytest.mark.asyncio\nasync def test_expires_after_round_trip(datasette):\n    \"\"\"Tokens with expires_after should include token_expires in the actor.\"\"\"\n    token = await datasette.create_token(\n        \"test_actor\", handler=\"signed\", expires_after=3600\n    )\n    actor = await datasette.verify_token(token)\n    assert actor is not None\n    assert actor[\"id\"] == \"test_actor\"\n    assert \"token_expires\" in actor\n\n\n@pytest.mark.asyncio\nasync def test_signed_tokens_disabled():\n    \"\"\"create_token and verify_token should fail/skip when signed tokens are disabled.\"\"\"\n    ds = Datasette(settings={\"allow_signed_tokens\": False})\n    with pytest.raises(ValueError, match=\"Signed tokens are not enabled\"):\n        await ds.create_token(\"test_actor\", handler=\"signed\")\n    # verify_token should return None rather than raising\n    assert await ds.verify_token(\"dstok_anything\") is None\n"
  },
  {
    "path": "tests/test_tracer.py",
    "content": "import pytest\nfrom .fixtures import make_app_client\n\n\n@pytest.mark.parametrize(\"trace_debug\", (True, False))\ndef test_trace(trace_debug):\n    with make_app_client(settings={\"trace_debug\": trace_debug}) as client:\n        response = client.get(\"/fixtures/simple_primary_key.json?_trace=1\")\n        assert response.status == 200\n\n    data = response.json\n    if not trace_debug:\n        assert \"_trace\" not in data\n        return\n\n    assert \"_trace\" in data\n    trace_info = data[\"_trace\"]\n    assert isinstance(trace_info[\"request_duration_ms\"], float)\n    assert isinstance(trace_info[\"sum_trace_duration_ms\"], float)\n    assert isinstance(trace_info[\"num_traces\"], int)\n    assert isinstance(trace_info[\"traces\"], list)\n    traces = trace_info[\"traces\"]\n    assert len(traces) == trace_info[\"num_traces\"]\n    for trace in traces:\n        assert isinstance(trace[\"type\"], str)\n        assert isinstance(trace[\"start\"], float)\n        assert isinstance(trace[\"end\"], float)\n        assert trace[\"duration_ms\"] == (trace[\"end\"] - trace[\"start\"]) * 1000\n        assert isinstance(trace[\"traceback\"], list)\n        assert isinstance(trace[\"database\"], str)\n        assert isinstance(trace[\"sql\"], str)\n        assert isinstance(trace.get(\"params\"), (list, dict, None.__class__))\n\n    sqls = [trace[\"sql\"] for trace in traces if \"sql\" in trace]\n    # There should be SQL statements from request handling in the trace.\n    # Note: CREATE TABLE, INSERT OR REPLACE, executescript, and executemany\n    # are not expected here because internal tables are now created and\n    # populated during invoke_startup(), before the request is traced.\n    assert any(sql.startswith(\"select \") for sql in sqls), \"No select statements traced\"\n\n\ndef test_trace_silently_fails_for_large_page():\n    # Max HTML size is 256KB\n    with make_app_client(settings={\"trace_debug\": True}) as client:\n        # Small response should have trace\n        small_response = client.get(\"/fixtures/simple_primary_key.json?_trace=1\")\n        assert small_response.status == 200\n        assert \"_trace\" in small_response.json\n\n        # Big response should not\n        big_response = client.get(\n            \"/fixtures/-/query.json\",\n            params={\"_trace\": 1, \"sql\": \"select zeroblob(1024 * 256)\"},\n        )\n        assert big_response.status == 200\n        assert \"_trace\" not in big_response.json\n\n\ndef test_trace_query_errors():\n    with make_app_client(settings={\"trace_debug\": True}) as client:\n        response = client.get(\n            \"/fixtures/-/query.json\",\n            params={\"_trace\": 1, \"sql\": \"select * from non_existent_table\"},\n        )\n        assert response.status == 400\n\n    data = response.json\n    assert \"_trace\" in data\n    trace_info = data[\"_trace\"]\n    assert trace_info[\"traces\"][-1][\"error\"] == \"no such table: non_existent_table\"\n\n\ndef test_trace_parallel_queries():\n    with make_app_client(settings={\"trace_debug\": True}) as client:\n        response = client.get(\"/parallel-queries?_trace=1\")\n        assert response.status == 200\n\n    data = response.json\n    assert data[\"one\"] == 1\n    assert data[\"two\"] == 2\n    trace_info = data[\"_trace\"]\n    traces = [trace for trace in trace_info[\"traces\"] if \"sql\" in trace]\n    one, two = traces\n    # \"two\" should have started before \"one\" ended\n    assert two[\"start\"] < one[\"end\"]\n"
  },
  {
    "path": "tests/test_utils.py",
    "content": "\"\"\"\nTests for various datasette helper functions.\n\"\"\"\n\nfrom datasette.app import Datasette\nfrom datasette import utils\nfrom datasette.utils.asgi import Request\nfrom datasette.utils.sqlite import sqlite3\nimport json\nimport os\nimport pathlib\nimport pytest\nimport tempfile\nfrom unittest.mock import patch\n\n\n@pytest.mark.parametrize(\n    \"path,expected\",\n    [\n        (\"foo\", [\"foo\"]),\n        (\"foo,bar\", [\"foo\", \"bar\"]),\n        (\"123,433,112\", [\"123\", \"433\", \"112\"]),\n        (\"123~2C433,112\", [\"123,433\", \"112\"]),\n        (\"123~2F433~2F112\", [\"123/433/112\"]),\n    ],\n)\ndef test_urlsafe_components(path, expected):\n    assert expected == utils.urlsafe_components(path)\n\n\n@pytest.mark.parametrize(\n    \"path,added_args,expected\",\n    [\n        (\"/foo\", {\"bar\": 1}, \"/foo?bar=1\"),\n        (\"/foo?bar=1\", {\"baz\": 2}, \"/foo?bar=1&baz=2\"),\n        (\"/foo?bar=1&bar=2\", {\"baz\": 3}, \"/foo?bar=1&bar=2&baz=3\"),\n        (\"/foo?bar=1\", {\"bar\": None}, \"/foo\"),\n        # Test order is preserved\n        (\n            \"/?_facet=prim_state&_facet=area_name\",\n            ((\"prim_state\", \"GA\"),),\n            \"/?_facet=prim_state&_facet=area_name&prim_state=GA\",\n        ),\n        (\n            \"/?_facet=state&_facet=city&state=MI\",\n            ((\"city\", \"Detroit\"),),\n            \"/?_facet=state&_facet=city&state=MI&city=Detroit\",\n        ),\n        (\n            \"/?_facet=state&_facet=city\",\n            ((\"_facet\", \"planet_int\"),),\n            \"/?_facet=state&_facet=city&_facet=planet_int\",\n        ),\n    ],\n)\ndef test_path_with_added_args(path, added_args, expected):\n    request = Request.fake(path)\n    actual = utils.path_with_added_args(request, added_args)\n    assert expected == actual\n\n\n@pytest.mark.parametrize(\n    \"path,args,expected\",\n    [\n        (\"/foo?bar=1\", {\"bar\"}, \"/foo\"),\n        (\"/foo?bar=1&baz=2\", {\"bar\"}, \"/foo?baz=2\"),\n        (\"/foo?bar=1&bar=2&bar=3\", {\"bar\": \"2\"}, \"/foo?bar=1&bar=3\"),\n    ],\n)\ndef test_path_with_removed_args(path, args, expected):\n    request = Request.fake(path)\n    actual = utils.path_with_removed_args(request, args)\n    assert expected == actual\n    # Run the test again but this time use the path= argument\n    request = Request.fake(\"/\")\n    actual = utils.path_with_removed_args(request, args, path=path)\n    assert expected == actual\n\n\n@pytest.mark.parametrize(\n    \"path,args,expected\",\n    [\n        (\"/foo?bar=1\", {\"bar\": 2}, \"/foo?bar=2\"),\n        (\"/foo?bar=1&baz=2\", {\"bar\": None}, \"/foo?baz=2\"),\n    ],\n)\ndef test_path_with_replaced_args(path, args, expected):\n    request = Request.fake(path)\n    actual = utils.path_with_replaced_args(request, args)\n    assert expected == actual\n\n\n@pytest.mark.parametrize(\n    \"row,pks,expected_path\",\n    [\n        ({\"A\": \"foo\", \"B\": \"bar\"}, [\"A\", \"B\"], \"foo,bar\"),\n        ({\"A\": \"f,o\", \"B\": \"bar\"}, [\"A\", \"B\"], \"f~2Co,bar\"),\n        ({\"A\": 123}, [\"A\"], \"123\"),\n        (\n            utils.CustomRow(\n                [\"searchable_id\", \"tag\"],\n                [\n                    (\"searchable_id\", {\"value\": 1, \"label\": \"1\"}),\n                    (\"tag\", {\"value\": \"feline\", \"label\": \"feline\"}),\n                ],\n            ),\n            [\"searchable_id\", \"tag\"],\n            \"1,feline\",\n        ),\n    ],\n)\ndef test_path_from_row_pks(row, pks, expected_path):\n    actual_path = utils.path_from_row_pks(row, pks, False)\n    assert expected_path == actual_path\n\n\n@pytest.mark.parametrize(\n    \"obj,expected\",\n    [\n        (\n            {\n                \"Description\": \"Soft drinks\",\n                \"Picture\": b\"\\x15\\x1c\\x02\\xc7\\xad\\x05\\xfe\",\n                \"CategoryID\": 1,\n            },\n            \"\"\"\n        {\"CategoryID\": 1, \"Description\": \"Soft drinks\", \"Picture\": {\"$base64\": true, \"encoded\": \"FRwCx60F/g==\"}}\n    \"\"\".strip(),\n        )\n    ],\n)\ndef test_custom_json_encoder(obj, expected):\n    actual = json.dumps(obj, cls=utils.CustomJSONEncoder, sort_keys=True)\n    assert expected == actual\n\n\n@pytest.mark.parametrize(\n    \"bad_sql\",\n    [\n        \"update blah;\",\n        \"-- sql comment to skip\\nupdate blah;\",\n        \"update blah set some_column='# Hello there\\n\\n* This is a list\\n* of items\\n--\\n[And a link](https://github.com/simonw/datasette-render-markdown).'\\nas demo_markdown\",\n        \"PRAGMA case_sensitive_like = true\",\n        \"SELECT * FROM pragma_not_on_allow_list('idx52')\",\n        \"/* This comment is not valid. select 1\",\n        \"/**/\\nupdate foo set bar = 1\\n/* test */ select 1\",\n    ],\n)\ndef test_validate_sql_select_bad(bad_sql):\n    with pytest.raises(utils.InvalidSql):\n        utils.validate_sql_select(bad_sql)\n\n\n@pytest.mark.parametrize(\n    \"good_sql\",\n    [\n        \"select count(*) from airports\",\n        \"select foo from bar\",\n        \"--sql comment to skip\\nselect foo from bar\",\n        \"select '# Hello there\\n\\n* This is a list\\n* of items\\n--\\n[And a link](https://github.com/simonw/datasette-render-markdown).'\\nas demo_markdown\",\n        \"select 1 + 1\",\n        \"explain select 1 + 1\",\n        \"explain\\nselect 1 + 1\",\n        \"explain query plan select 1 + 1\",\n        \"explain  query  plan\\nselect 1 + 1\",\n        \"SELECT\\nblah FROM foo\",\n        \"WITH RECURSIVE cnt(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM cnt LIMIT 10) SELECT x FROM cnt;\",\n        \"explain WITH RECURSIVE cnt(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM cnt LIMIT 10) SELECT x FROM cnt;\",\n        \"explain query plan WITH RECURSIVE cnt(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM cnt LIMIT 10) SELECT x FROM cnt;\",\n        \"SELECT * FROM pragma_index_info('idx52')\",\n        \"select * from pragma_table_xinfo('table')\",\n        # Various types of comment\n        \"-- comment\\nselect 1\",\n        \"-- one line\\n  -- two line\\nselect 1\",\n        \"  /* comment */\\nselect 1\",\n        \"  /* comment */select 1\",\n        \"/* comment */\\n -- another\\n /* one more */ select 1\",\n        \"/* This comment \\n has multiple lines */\\nselect 1\",\n    ],\n)\ndef test_validate_sql_select_good(good_sql):\n    utils.validate_sql_select(good_sql)\n\n\n@pytest.mark.parametrize(\"open_quote,close_quote\", [('\"', '\"'), (\"[\", \"]\")])\ndef test_detect_fts(open_quote, close_quote):\n    sql = \"\"\"\n    CREATE TABLE \"Dumb_Table\" (\n      \"TreeID\" INTEGER,\n      \"qSpecies\" TEXT\n    );\n    CREATE TABLE \"Street_Tree_List\" (\n      \"TreeID\" INTEGER,\n      \"qSpecies\" TEXT,\n      \"qAddress\" TEXT,\n      \"SiteOrder\" INTEGER,\n      \"qSiteInfo\" TEXT,\n      \"PlantType\" TEXT,\n      \"qCaretaker\" TEXT\n    );\n    CREATE VIEW Test_View AS SELECT * FROM Dumb_Table;\n    CREATE VIRTUAL TABLE {open}Street_Tree_List_fts{close} USING FTS4 (\"qAddress\", \"qCaretaker\", \"qSpecies\", content={open}Street_Tree_List{close});\n    CREATE VIRTUAL TABLE r USING rtree(a, b, c);\n    \"\"\".format(open=open_quote, close=close_quote)\n    conn = utils.sqlite3.connect(\":memory:\")\n    conn.executescript(sql)\n    assert None is utils.detect_fts(conn, \"Dumb_Table\")\n    assert None is utils.detect_fts(conn, \"Test_View\")\n    assert None is utils.detect_fts(conn, \"r\")\n    assert \"Street_Tree_List_fts\" == utils.detect_fts(conn, \"Street_Tree_List\")\n\n\n@pytest.mark.parametrize(\"table\", (\"regular\", \"has'single quote\"))\ndef test_detect_fts_different_table_names(table):\n    sql = \"\"\"\n    CREATE TABLE [{table}] (\n      \"TreeID\" INTEGER,\n      \"qSpecies\" TEXT\n    );\n    CREATE VIRTUAL TABLE [{table}_fts] USING FTS4 (\"qSpecies\", content=\"{table}\");\n    \"\"\".format(table=table)\n    conn = utils.sqlite3.connect(\":memory:\")\n    conn.executescript(sql)\n    assert \"{table}_fts\".format(table=table) == utils.detect_fts(conn, table)\n\n\n@pytest.mark.parametrize(\n    \"url,expected\",\n    [\n        (\"http://www.google.com/\", True),\n        (\"https://example.com/\", True),\n        (\"www.google.com\", False),\n        (\"http://www.google.com/ is a search engine\", False),\n    ],\n)\ndef test_is_url(url, expected):\n    assert expected == utils.is_url(url)\n\n\n@pytest.mark.parametrize(\n    \"s,expected\",\n    [\n        (\"simple\", \"simple\"),\n        (\"MixedCase\", \"MixedCase\"),\n        (\"-no-leading-hyphens\", \"no-leading-hyphens-65bea6\"),\n        (\"_no-leading-underscores\", \"no-leading-underscores-b921bc\"),\n        (\"no spaces\", \"no-spaces-7088d7\"),\n        (\"-\", \"336d5e\"),\n        (\"no $ characters\", \"no--characters-59e024\"),\n    ],\n)\ndef test_to_css_class(s, expected):\n    assert expected == utils.to_css_class(s)\n\n\ndef test_temporary_docker_directory_uses_hard_link():\n    with tempfile.TemporaryDirectory() as td:\n        os.chdir(td)\n        with open(\"hello\", \"w\") as fp:\n            fp.write(\"world\")\n        # Default usage of this should use symlink\n        with utils.temporary_docker_directory(\n            files=[\"hello\"],\n            name=\"t\",\n            metadata=None,\n            extra_options=None,\n            branch=None,\n            template_dir=None,\n            plugins_dir=None,\n            static=[],\n            install=[],\n            spatialite=False,\n            version_note=None,\n            secret=\"secret\",\n        ) as temp_docker:\n            hello = os.path.join(temp_docker, \"hello\")\n            with open(hello) as fp:\n                assert \"world\" == fp.read()\n            # It should be a hard link\n            assert 2 == os.stat(hello).st_nlink\n\n\n@patch(\"os.link\")\ndef test_temporary_docker_directory_uses_copy_if_hard_link_fails(mock_link):\n    # Copy instead if os.link raises OSError (normally due to different device)\n    mock_link.side_effect = OSError\n    with tempfile.TemporaryDirectory() as td:\n        os.chdir(td)\n        with open(\"hello\", \"w\") as fp:\n            fp.write(\"world\")\n        # Default usage of this should use symlink\n        with utils.temporary_docker_directory(\n            files=[\"hello\"],\n            name=\"t\",\n            metadata=None,\n            extra_options=None,\n            branch=None,\n            template_dir=None,\n            plugins_dir=None,\n            static=[],\n            install=[],\n            spatialite=False,\n            version_note=None,\n            secret=None,\n        ) as temp_docker:\n            hello = os.path.join(temp_docker, \"hello\")\n            with open(hello) as fp:\n                assert \"world\" == fp.read()\n            # It should be a copy, not a hard link\n            assert 1 == os.stat(hello).st_nlink\n\n\ndef test_temporary_docker_directory_quotes_args():\n    with tempfile.TemporaryDirectory() as td:\n        os.chdir(td)\n        with open(\"hello\", \"w\") as fp:\n            fp.write(\"world\")\n        with utils.temporary_docker_directory(\n            files=[\"hello\"],\n            name=\"t\",\n            metadata=None,\n            extra_options=\"--$HOME\",\n            branch=None,\n            template_dir=None,\n            plugins_dir=None,\n            static=[],\n            install=[],\n            spatialite=False,\n            version_note=\"$PWD\",\n            secret=\"secret\",\n        ) as temp_docker:\n            df = os.path.join(temp_docker, \"Dockerfile\")\n            with open(df) as fp:\n                df_contents = fp.read()\n            assert \"'$PWD'\" in df_contents\n            assert \"'--$HOME'\" in df_contents\n            assert \"ENV DATASETTE_SECRET 'secret'\" in df_contents\n\n\ndef test_compound_keys_after_sql():\n    assert \"((a > :p0))\" == utils.compound_keys_after_sql([\"a\"])\n    assert \"\"\"\n((a > :p0)\n  or\n(a = :p0 and b > :p1))\n    \"\"\".strip() == utils.compound_keys_after_sql([\"a\", \"b\"])\n    assert \"\"\"\n((a > :p0)\n  or\n(a = :p0 and b > :p1)\n  or\n(a = :p0 and b = :p1 and c > :p2))\n    \"\"\".strip() == utils.compound_keys_after_sql([\"a\", \"b\", \"c\"])\n\n\ndef test_table_columns():\n    conn = sqlite3.connect(\":memory:\")\n    conn.executescript(\"\"\"\n    create table places (id integer primary key, name text, bob integer)\n    \"\"\")\n    assert [\"id\", \"name\", \"bob\"] == utils.table_columns(conn, \"places\")\n\n\n@pytest.mark.parametrize(\n    \"path,format,extra_qs,expected\",\n    [\n        (\"/foo?sql=select+1\", \"csv\", {}, \"/foo.csv?sql=select+1\"),\n        (\"/foo?sql=select+1\", \"json\", {}, \"/foo.json?sql=select+1\"),\n        (\"/foo/bar\", \"json\", {}, \"/foo/bar.json\"),\n        (\"/foo/bar\", \"csv\", {}, \"/foo/bar.csv\"),\n        (\"/foo/bar\", \"csv\", {\"_dl\": 1}, \"/foo/bar.csv?_dl=1\"),\n        (\n            \"/sf-trees/Street_Tree_List?_search=cherry&_size=1000\",\n            \"csv\",\n            {\"_dl\": 1},\n            \"/sf-trees/Street_Tree_List.csv?_search=cherry&_size=1000&_dl=1\",\n        ),\n    ],\n)\ndef test_path_with_format(path, format, extra_qs, expected):\n    request = Request.fake(path)\n    actual = utils.path_with_format(request=request, format=format, extra_qs=extra_qs)\n    assert expected == actual\n\n\n@pytest.mark.parametrize(\n    \"bytes,expected\",\n    [\n        (120, \"120 bytes\"),\n        (1024, \"1.0 KB\"),\n        (1024 * 1024, \"1.0 MB\"),\n        (1024 * 1024 * 1024, \"1.0 GB\"),\n        (1024 * 1024 * 1024 * 1.3, \"1.3 GB\"),\n        (1024 * 1024 * 1024 * 1024, \"1.0 TB\"),\n    ],\n)\ndef test_format_bytes(bytes, expected):\n    assert expected == utils.format_bytes(bytes)\n\n\n@pytest.mark.parametrize(\n    \"query,expected\",\n    [\n        (\"dog\", '\"dog\"'),\n        (\"cat,\", '\"cat,\"'),\n        (\"cat dog\", '\"cat\" \"dog\"'),\n        # If a phrase is already double quoted, leave it so\n        ('\"cat dog\"', '\"cat dog\"'),\n        ('\"cat dog\" fish', '\"cat dog\" \"fish\"'),\n        # Sensibly handle unbalanced double quotes\n        ('cat\"', '\"cat\"'),\n        ('\"cat dog\" \"fish', '\"cat dog\" \"fish\"'),\n    ],\n)\ndef test_escape_fts(query, expected):\n    assert expected == utils.escape_fts(query)\n\n\n@pytest.mark.parametrize(\n    \"input,expected\",\n    [\n        (\"dog\", \"dog\"),\n        ('dateutil_parse(\"1/2/2020\")', r\"dateutil_parse(\\0000221/2/2020\\000022)\"),\n        (\"this\\r\\nand\\r\\nthat\", r\"this\\00000Aand\\00000Athat\"),\n    ],\n)\ndef test_escape_css_string(input, expected):\n    assert expected == utils.escape_css_string(input)\n\n\ndef test_check_connection_spatialite_raises():\n    path = str(pathlib.Path(__file__).parent / \"spatialite.db\")\n    conn = sqlite3.connect(path)\n    with pytest.raises(utils.SpatialiteConnectionProblem):\n        utils.check_connection(conn)\n\n\ndef test_check_connection_passes():\n    conn = sqlite3.connect(\":memory:\")\n    utils.check_connection(conn)\n\n\ndef test_call_with_supported_arguments():\n    def foo(a, b):\n        return f\"{a}+{b}\"\n\n    assert \"1+2\" == utils.call_with_supported_arguments(foo, a=1, b=2)\n    assert \"1+2\" == utils.call_with_supported_arguments(foo, a=1, b=2, c=3)\n\n    with pytest.raises(TypeError):\n        utils.call_with_supported_arguments(foo, a=1)\n\n\n@pytest.mark.parametrize(\n    \"data,should_raise\",\n    [\n        ([[\"foo\", \"bar\"], [\"foo\", \"baz\"]], False),\n        ([(\"foo\", \"bar\"), (\"foo\", \"baz\")], False),\n        (([\"foo\", \"bar\"], [\"foo\", \"baz\"]), False),\n        ([[\"foo\", \"bar\"], [\"foo\", \"baz\", \"bax\"]], True),\n        ({\"foo\": [\"bar\", \"baz\"]}, False),\n        ({\"foo\": (\"bar\", \"baz\")}, False),\n        ({\"foo\": \"bar\"}, True),\n    ],\n)\ndef test_multi_params(data, should_raise):\n    if should_raise:\n        with pytest.raises(AssertionError):\n            utils.MultiParams(data)\n        return\n    p1 = utils.MultiParams(data)\n    assert \"bar\" == p1[\"foo\"]\n    assert [\"bar\", \"baz\"] == list(p1.getlist(\"foo\"))\n\n\n@pytest.mark.parametrize(\n    \"actor,allow,expected\",\n    [\n        # Default is to allow:\n        (None, None, True),\n        # {} means deny-all:\n        (None, {}, False),\n        ({\"id\": \"root\"}, {}, False),\n        # true means allow-all\n        ({\"id\": \"root\"}, True, True),\n        (None, True, True),\n        # false means deny-all\n        ({\"id\": \"root\"}, False, False),\n        (None, False, False),\n        # Special case for \"unauthenticated\": true\n        (None, {\"unauthenticated\": True}, True),\n        (None, {\"unauthenticated\": False}, False),\n        # Match on just one property:\n        (None, {\"id\": \"root\"}, False),\n        ({\"id\": \"root\"}, None, True),\n        ({\"id\": \"simon\", \"staff\": True}, {\"staff\": True}, True),\n        ({\"id\": \"simon\", \"staff\": False}, {\"staff\": True}, False),\n        # Special \"*\" value for any key:\n        ({\"id\": \"root\"}, {\"id\": \"*\"}, True),\n        ({}, {\"id\": \"*\"}, False),\n        ({\"name\": \"root\"}, {\"id\": \"*\"}, False),\n        # Supports single strings or list of values:\n        ({\"id\": \"root\"}, {\"id\": \"bob\"}, False),\n        ({\"id\": \"root\"}, {\"id\": [\"bob\"]}, False),\n        ({\"id\": \"root\"}, {\"id\": \"root\"}, True),\n        ({\"id\": \"root\"}, {\"id\": [\"root\"]}, True),\n        # Any matching role will work:\n        ({\"id\": \"garry\", \"roles\": [\"staff\", \"dev\"]}, {\"roles\": [\"staff\"]}, True),\n        ({\"id\": \"garry\", \"roles\": [\"staff\", \"dev\"]}, {\"roles\": [\"dev\"]}, True),\n        ({\"id\": \"garry\", \"roles\": [\"staff\", \"dev\"]}, {\"roles\": [\"otter\"]}, False),\n        ({\"id\": \"garry\", \"roles\": [\"staff\", \"dev\"]}, {\"roles\": [\"dev\", \"otter\"]}, True),\n        ({\"id\": \"garry\", \"roles\": []}, {\"roles\": [\"staff\"]}, False),\n        ({\"id\": \"garry\"}, {\"roles\": [\"staff\"]}, False),\n        # Any single matching key works:\n        ({\"id\": \"root\"}, {\"bot_id\": \"my-bot\", \"id\": [\"root\"]}, True),\n    ],\n)\ndef test_actor_matches_allow(actor, allow, expected):\n    assert expected == utils.actor_matches_allow(actor, allow)\n\n\n@pytest.mark.parametrize(\n    \"config,expected\",\n    [\n        ({\"foo\": \"bar\"}, {\"foo\": \"bar\"}),\n        ({\"$env\": \"FOO\"}, \"x\"),\n        ({\"k\": {\"$env\": \"FOO\"}}, {\"k\": \"x\"}),\n        ([{\"k\": {\"$env\": \"FOO\"}}, {\"z\": {\"$env\": \"FOO\"}}], [{\"k\": \"x\"}, {\"z\": \"x\"}]),\n        ({\"k\": [{\"in_a_list\": {\"$env\": \"FOO\"}}]}, {\"k\": [{\"in_a_list\": \"x\"}]}),\n    ],\n)\ndef test_resolve_env_secrets(config, expected):\n    assert expected == utils.resolve_env_secrets(config, {\"FOO\": \"x\"})\n\n\n@pytest.mark.parametrize(\n    \"actor,expected\",\n    [\n        ({\"id\": \"blah\"}, \"blah\"),\n        ({\"id\": \"blah\", \"login\": \"l\"}, \"l\"),\n        ({\"id\": \"blah\", \"login\": \"l\"}, \"l\"),\n        ({\"id\": \"blah\", \"login\": \"l\", \"username\": \"u\"}, \"u\"),\n        ({\"login\": \"l\", \"name\": \"n\"}, \"n\"),\n        (\n            {\"id\": \"blah\", \"login\": \"l\", \"username\": \"u\", \"name\": \"n\", \"display\": \"d\"},\n            \"d\",\n        ),\n        ({\"weird\": \"shape\"}, \"{'weird': 'shape'}\"),\n    ],\n)\ndef test_display_actor(actor, expected):\n    assert expected == utils.display_actor(actor)\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"dbs,expected_path\",\n    [\n        ([\"one_table\"], \"/one/one\"),\n        ([\"two_tables\"], \"/two\"),\n        ([\"one_table\", \"two_tables\"], \"/\"),\n    ],\n)\nasync def test_initial_path_for_datasette(tmp_path_factory, dbs, expected_path):\n    db_dir = tmp_path_factory.mktemp(\"dbs\")\n    one_table = str(db_dir / \"one.db\")\n    sqlite3.connect(one_table).execute(\"create table one (id integer primary key)\")\n    two_tables = str(db_dir / \"two.db\")\n    sqlite3.connect(two_tables).execute(\"create table two (id integer primary key)\")\n    sqlite3.connect(two_tables).execute(\"create table three (id integer primary key)\")\n    datasette = Datasette(\n        [{\"one_table\": one_table, \"two_tables\": two_tables}[db] for db in dbs]\n    )\n    path = await utils.initial_path_for_datasette(datasette)\n    assert path == expected_path\n\n\n@pytest.mark.parametrize(\n    \"content,expected\",\n    (\n        (\"title: Hello\", {\"title\": \"Hello\"}),\n        ('{\"title\": \"Hello\"}', {\"title\": \"Hello\"}),\n        (\"{{ this }} is {{ bad }}\", None),\n    ),\n)\ndef test_parse_metadata(content, expected):\n    if expected is None:\n        with pytest.raises(utils.BadMetadataError):\n            utils.parse_metadata(content)\n    else:\n        assert utils.parse_metadata(content) == expected\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"sql,expected\",\n    (\n        (\"select 1\", []),\n        (\"select 1 + :one\", [\"one\"]),\n        (\"select 1 + :one + :two\", [\"one\", \"two\"]),\n        (\"select 'bob' || '0:00' || :cat\", [\"cat\"]),\n        (\"select this is invalid :one, :two, :three\", [\"one\", \"two\", \"three\"]),\n    ),\n)\n@pytest.mark.parametrize(\"use_async_version\", (False, True))\nasync def test_named_parameters(sql, expected, use_async_version):\n    ds = Datasette([], memory=True)\n    db = ds.get_database(\"_memory\")\n    if use_async_version:\n        params = await utils.derive_named_parameters(db, sql)\n    else:\n        params = utils.named_parameters(sql)\n    assert params == expected\n\n\n@pytest.mark.parametrize(\n    \"original,expected\",\n    (\n        (\"abc\", \"abc\"),\n        (\"/foo/bar\", \"~2Ffoo~2Fbar\"),\n        (\"/-/bar\", \"~2F-~2Fbar\"),\n        (\"-/db-/table.csv\", \"-~2Fdb-~2Ftable~2Ecsv\"),\n        (r\"%~-/\", \"~25~7E-~2F\"),\n        (\"~25~7E~2D~2F\", \"~7E25~7E7E~7E2D~7E2F\"),\n        (\"with space\", \"with+space\"),\n    ),\n)\ndef test_tilde_encoding(original, expected):\n    actual = utils.tilde_encode(original)\n    assert actual == expected\n    # And test round-trip\n    assert original == utils.tilde_decode(actual)\n\n\n@pytest.mark.parametrize(\n    \"url,length,expected\",\n    (\n        (\"https://example.com/\", 5, \"http…\"),\n        (\"https://example.com/foo/bar\", 15, \"https://exampl…\"),\n        (\"https://example.com/foo/bar/baz.jpg\", 30, \"https://example.com/foo/ba….jpg\"),\n        # Extensions longer than 4 characters are not treated specially:\n        (\"https://example.com/foo/bar/baz.jpeg2\", 30, \"https://example.com/foo/bar/b…\"),\n        (\n            \"https://example.com/foo/bar/baz.jpeg2\",\n            None,\n            \"https://example.com/foo/bar/baz.jpeg2\",\n        ),\n    ),\n)\ndef test_truncate_url(url, length, expected):\n    actual = utils.truncate_url(url, length)\n    assert actual == expected\n\n\n@pytest.mark.parametrize(\n    \"pairs,expected\",\n    (\n        # Simple nested objects\n        ([(\"a\", \"b\")], {\"a\": \"b\"}),\n        ([(\"a.b\", \"c\")], {\"a\": {\"b\": \"c\"}}),\n        # JSON literals\n        ([(\"a.b\", \"true\")], {\"a\": {\"b\": True}}),\n        ([(\"a.b\", \"false\")], {\"a\": {\"b\": False}}),\n        ([(\"a.b\", \"null\")], {\"a\": {\"b\": None}}),\n        ([(\"a.b\", \"1\")], {\"a\": {\"b\": 1}}),\n        ([(\"a.b\", \"1.1\")], {\"a\": {\"b\": 1.1}}),\n        # Nested JSON literals\n        ([(\"a.b\", '{\"foo\": \"bar\"}')], {\"a\": {\"b\": {\"foo\": \"bar\"}}}),\n        ([(\"a.b\", \"[1, 2, 3]\")], {\"a\": {\"b\": [1, 2, 3]}}),\n        # JSON strings are preserved\n        ([(\"a.b\", '\"true\"')], {\"a\": {\"b\": \"true\"}}),\n        ([(\"a.b\", '\"[1, 2, 3]\"')], {\"a\": {\"b\": \"[1, 2, 3]\"}}),\n        # Later keys over-ride the previous\n        (\n            [\n                (\"a\", \"b\"),\n                (\"a.b\", \"c\"),\n            ],\n            {\"a\": {\"b\": \"c\"}},\n        ),\n        (\n            [\n                (\"settings.trace_debug\", \"true\"),\n                (\"plugins.datasette-ripgrep.path\", \"/etc\"),\n                (\"settings.trace_debug\", \"false\"),\n            ],\n            {\n                \"settings\": {\n                    \"trace_debug\": False,\n                },\n                \"plugins\": {\n                    \"datasette-ripgrep\": {\n                        \"path\": \"/etc\",\n                    }\n                },\n            },\n        ),\n    ),\n)\ndef test_pairs_to_nested_config(pairs, expected):\n    actual = utils.pairs_to_nested_config(pairs)\n    assert actual == expected\n\n\n@pytest.mark.asyncio\nasync def test_calculate_etag(tmp_path):\n    path = tmp_path / \"test.txt\"\n    path.write_text(\"hello\")\n    etag = '\"5d41402abc4b2a76b9719d911017c592\"'\n    assert etag == await utils.calculate_etag(path)\n    assert utils._etag_cache[path] == etag\n    utils._etag_cache[path] = \"hash\"\n    assert \"hash\" == await utils.calculate_etag(path)\n    utils._etag_cache.clear()\n\n\n@pytest.mark.parametrize(\n    \"dict1,dict2,expected\",\n    [\n        # Basic update\n        ({\"a\": 1, \"b\": 2}, {\"b\": 3, \"c\": 4}, {\"a\": 1, \"b\": 3, \"c\": 4}),\n        # Nested dictionary update\n        (\n            {\"a\": 1, \"b\": {\"x\": 10, \"y\": 20}},\n            {\"b\": {\"y\": 30, \"z\": 40}},\n            {\"a\": 1, \"b\": {\"x\": 10, \"y\": 30, \"z\": 40}},\n        ),\n        # Deep nested update\n        (\n            {\"a\": {\"b\": {\"c\": 1}}},\n            {\"a\": {\"b\": {\"d\": 2}}},\n            {\"a\": {\"b\": {\"c\": 1, \"d\": 2}}},\n        ),\n        # Update with mixed types\n        (\n            {\"a\": 1, \"b\": {\"x\": 10}},\n            {\"b\": {\"y\": 20}, \"c\": [1, 2, 3]},\n            {\"a\": 1, \"b\": {\"x\": 10, \"y\": 20}, \"c\": [1, 2, 3]},\n        ),\n    ],\n)\ndef test_deep_dict_update(dict1, dict2, expected):\n    result = utils.deep_dict_update(dict1, dict2)\n    assert result == expected\n    # Check that the original dict1 was modified\n    assert dict1 == expected\n"
  },
  {
    "path": "tests/test_utils_check_callable.py",
    "content": "from datasette.utils.check_callable import check_callable\nimport pytest\n\n\nclass AsyncClass:\n    async def __call__(self):\n        pass\n\n\nclass NotAsyncClass:\n    def __call__(self):\n        pass\n\n\nclass ClassNoCall:\n    pass\n\n\nasync def async_func():\n    pass\n\n\ndef non_async_func():\n    pass\n\n\n@pytest.mark.parametrize(\n    \"obj,expected_is_callable,expected_is_async_callable\",\n    (\n        (async_func, True, True),\n        (non_async_func, True, False),\n        (AsyncClass(), True, True),\n        (NotAsyncClass(), True, False),\n        (ClassNoCall(), False, False),\n        (AsyncClass, True, False),\n        (NotAsyncClass, True, False),\n        (ClassNoCall, True, False),\n        (\"\", False, False),\n        (1, False, False),\n        (str, True, False),\n    ),\n)\ndef test_check_callable(obj, expected_is_callable, expected_is_async_callable):\n    status = check_callable(obj)\n    assert status.is_callable == expected_is_callable\n    assert status.is_async_callable == expected_is_async_callable\n"
  },
  {
    "path": "tests/test_utils_permissions.py",
    "content": "import pytest\nfrom datasette.app import Datasette\nfrom datasette.permissions import PermissionSQL\nfrom datasette.utils.permissions import resolve_permissions_from_catalog\nfrom typing import Callable, List\n\n\n@pytest.fixture\ndef db():\n    ds = Datasette()\n    import tempfile\n    from datasette.database import Database\n\n    path = tempfile.mktemp(suffix=\"demo.db\")\n    db = ds.add_database(Database(ds, path=path))\n    return db\n\n\nNO_RULES_SQL = (\n    \"SELECT NULL AS parent, NULL AS child, NULL AS allow, NULL AS reason WHERE 0\"\n)\n\n\ndef plugin_allow_all_for_user(user: str) -> Callable[[str], PermissionSQL]:\n    def provider(action: str) -> PermissionSQL:\n        return PermissionSQL(\n            \"\"\"\n            SELECT NULL AS parent, NULL AS child, 1 AS allow,\n                   'global allow for ' || :allow_all_user || ' on ' || :allow_all_action AS reason\n            WHERE :actor_id = :allow_all_user\n            \"\"\",\n            {\"allow_all_user\": user, \"allow_all_action\": action},\n        )\n\n    return provider\n\n\ndef plugin_deny_specific_table(\n    user: str, parent: str, child: str\n) -> Callable[[str], PermissionSQL]:\n    def provider(action: str) -> PermissionSQL:\n        return PermissionSQL(\n            \"\"\"\n            SELECT :deny_specific_table_parent AS parent, :deny_specific_table_child AS child, 0 AS allow,\n                   'deny ' || :deny_specific_table_parent || '/' || :deny_specific_table_child || ' for ' || :deny_specific_table_user || ' on ' || :deny_specific_table_action AS reason\n            WHERE :actor_id = :deny_specific_table_user\n            \"\"\",\n            {\n                \"deny_specific_table_parent\": parent,\n                \"deny_specific_table_child\": child,\n                \"deny_specific_table_user\": user,\n                \"deny_specific_table_action\": action,\n            },\n        )\n\n    return provider\n\n\ndef plugin_org_policy_deny_parent(parent: str) -> Callable[[str], PermissionSQL]:\n    def provider(action: str) -> PermissionSQL:\n        return PermissionSQL(\n            \"\"\"\n            SELECT :org_policy_parent_deny_parent AS parent, NULL AS child, 0 AS allow,\n                   'org policy: parent ' || :org_policy_parent_deny_parent || ' denied on ' || :org_policy_parent_deny_action AS reason\n            \"\"\",\n            {\n                \"org_policy_parent_deny_parent\": parent,\n                \"org_policy_parent_deny_action\": action,\n            },\n        )\n\n    return provider\n\n\ndef plugin_allow_parent_for_user(\n    user: str, parent: str\n) -> Callable[[str], PermissionSQL]:\n    def provider(action: str) -> PermissionSQL:\n        return PermissionSQL(\n            \"\"\"\n            SELECT :allow_parent_parent AS parent, NULL AS child, 1 AS allow,\n                   'allow full parent for ' || :allow_parent_user || ' on ' || :allow_parent_action AS reason\n            WHERE :actor_id = :allow_parent_user\n            \"\"\",\n            {\n                \"allow_parent_parent\": parent,\n                \"allow_parent_user\": user,\n                \"allow_parent_action\": action,\n            },\n        )\n\n    return provider\n\n\ndef plugin_child_allow_for_user(\n    user: str, parent: str, child: str\n) -> Callable[[str], PermissionSQL]:\n    def provider(action: str) -> PermissionSQL:\n        return PermissionSQL(\n            \"\"\"\n            SELECT :allow_child_parent AS parent, :allow_child_child AS child, 1 AS allow,\n                   'allow child for ' || :allow_child_user || ' on ' || :allow_child_action AS reason\n            WHERE :actor_id = :allow_child_user\n            \"\"\",\n            {\n                \"allow_child_parent\": parent,\n                \"allow_child_child\": child,\n                \"allow_child_user\": user,\n                \"allow_child_action\": action,\n            },\n        )\n\n    return provider\n\n\ndef plugin_root_deny_for_all() -> Callable[[str], PermissionSQL]:\n    def provider(action: str) -> PermissionSQL:\n        return PermissionSQL(\n            \"\"\"\n            SELECT NULL AS parent, NULL AS child, 0 AS allow, 'root deny for all on ' || :root_deny_action AS reason\n            \"\"\",\n            {\"root_deny_action\": action},\n        )\n\n    return provider\n\n\ndef plugin_conflicting_same_child_rules(\n    user: str, parent: str, child: str\n) -> List[Callable[[str], PermissionSQL]]:\n    def allow_provider(action: str) -> PermissionSQL:\n        return PermissionSQL(\n            \"\"\"\n            SELECT :conflict_child_allow_parent AS parent, :conflict_child_allow_child AS child, 1 AS allow,\n                   'team grant at child for ' || :conflict_child_allow_user || ' on ' || :conflict_child_allow_action AS reason\n            WHERE :actor_id = :conflict_child_allow_user\n            \"\"\",\n            {\n                \"conflict_child_allow_parent\": parent,\n                \"conflict_child_allow_child\": child,\n                \"conflict_child_allow_user\": user,\n                \"conflict_child_allow_action\": action,\n            },\n        )\n\n    def deny_provider(action: str) -> PermissionSQL:\n        return PermissionSQL(\n            \"\"\"\n            SELECT :conflict_child_deny_parent AS parent, :conflict_child_deny_child AS child, 0 AS allow,\n                   'exception deny at child for ' || :conflict_child_deny_user || ' on ' || :conflict_child_deny_action AS reason\n            WHERE :actor_id = :conflict_child_deny_user\n            \"\"\",\n            {\n                \"conflict_child_deny_parent\": parent,\n                \"conflict_child_deny_child\": child,\n                \"conflict_child_deny_user\": user,\n                \"conflict_child_deny_action\": action,\n            },\n        )\n\n    return [allow_provider, deny_provider]\n\n\ndef plugin_allow_all_for_action(\n    user: str, allowed_action: str\n) -> Callable[[str], PermissionSQL]:\n    def provider(action: str) -> PermissionSQL:\n        if action != allowed_action:\n            return PermissionSQL(NO_RULES_SQL)\n        # Sanitize parameter names by replacing hyphens with underscores\n        param_prefix = action.replace(\"-\", \"_\")\n        return PermissionSQL(\n            f\"\"\"\n            SELECT NULL AS parent, NULL AS child, 1 AS allow,\n                   'global allow for ' || :{param_prefix}_user || ' on ' || :{param_prefix}_action AS reason\n            WHERE :actor_id = :{param_prefix}_user\n            \"\"\",\n            {f\"{param_prefix}_user\": user, f\"{param_prefix}_action\": action},\n        )\n\n    return provider\n\n\nVIEW_TABLE = \"view-table\"\n\n\n# ---------- Catalog DDL (from your schema) ----------\nCATALOG_DDL = \"\"\"\nCREATE TABLE IF NOT EXISTS catalog_databases (\n    database_name TEXT PRIMARY KEY,\n    path TEXT,\n    is_memory INTEGER,\n    schema_version INTEGER\n);\nCREATE TABLE IF NOT EXISTS catalog_tables (\n    database_name TEXT,\n    table_name TEXT,\n    rootpage INTEGER,\n    sql TEXT,\n    PRIMARY KEY (database_name, table_name),\n    FOREIGN KEY (database_name) REFERENCES catalog_databases(database_name)\n);\n\"\"\"\n\nPARENTS = [\"accounting\", \"hr\", \"analytics\"]\nSPECIALS = {\"accounting\": [\"sales\"], \"analytics\": [\"secret\"], \"hr\": []}\n\nTABLE_CANDIDATES_SQL = (\n    \"SELECT database_name AS parent, table_name AS child FROM catalog_tables\"\n)\nPARENT_CANDIDATES_SQL = (\n    \"SELECT database_name AS parent, NULL AS child FROM catalog_databases\"\n)\n\n\n# ---------- Helpers ----------\nasync def seed_catalog(db, per_parent: int = 10) -> None:\n    await db.execute_write_script(CATALOG_DDL)\n    # databases\n    db_rows = [(p, f\"/{p}.db\", 0, 1) for p in PARENTS]\n    await db.execute_write_many(\n        \"INSERT OR REPLACE INTO catalog_databases(database_name, path, is_memory, schema_version) VALUES (?,?,?,?)\",\n        db_rows,\n    )\n\n    # tables\n    def tables_for(parent: str, n: int):\n        base = [f\"table{i:02d}\" for i in range(1, n + 1)]\n        for s in SPECIALS.get(parent, []):\n            if s not in base:\n                base[0] = s\n        return base\n\n    table_rows = []\n    for p in PARENTS:\n        for t in tables_for(p, per_parent):\n            table_rows.append((p, t, 0, f\"CREATE TABLE {t} (id INTEGER PRIMARY KEY)\"))\n    await db.execute_write_many(\n        \"INSERT OR REPLACE INTO catalog_tables(database_name, table_name, rootpage, sql) VALUES (?,?,?,?)\",\n        table_rows,\n    )\n\n\ndef res_allowed(rows, parent=None):\n    return sorted(\n        r[\"resource\"]\n        for r in rows\n        if r[\"allow\"] == 1 and (parent is None or r[\"parent\"] == parent)\n    )\n\n\ndef res_denied(rows, parent=None):\n    return sorted(\n        r[\"resource\"]\n        for r in rows\n        if r[\"allow\"] == 0 and (parent is None or r[\"parent\"] == parent)\n    )\n\n\n# ---------- Tests ----------\n@pytest.mark.asyncio\nasync def test_alice_global_allow_with_specific_denies_catalog(db):\n    await seed_catalog(db)\n    plugins = [\n        plugin_allow_all_for_user(\"alice\"),\n        plugin_deny_specific_table(\"alice\", \"accounting\", \"sales\"),\n        plugin_org_policy_deny_parent(\"hr\"),\n    ]\n    rows = await resolve_permissions_from_catalog(\n        db,\n        {\"id\": \"alice\"},\n        plugins,\n        VIEW_TABLE,\n        TABLE_CANDIDATES_SQL,\n        implicit_deny=True,\n    )\n    # Alice can see everything except accounting/sales and hr/*\n    assert \"/accounting/sales\" in res_denied(rows)\n    for r in rows:\n        if r[\"parent\"] == \"hr\":\n            assert r[\"allow\"] == 0\n        elif r[\"resource\"] == \"/accounting/sales\":\n            assert r[\"allow\"] == 0\n        else:\n            assert r[\"allow\"] == 1\n\n\n@pytest.mark.asyncio\nasync def test_carol_parent_allow_but_child_conflict_deny_wins_catalog(db):\n    await seed_catalog(db)\n    plugins = [\n        plugin_org_policy_deny_parent(\"hr\"),\n        plugin_allow_parent_for_user(\"carol\", \"analytics\"),\n        *plugin_conflicting_same_child_rules(\"carol\", \"analytics\", \"secret\"),\n    ]\n    rows = await resolve_permissions_from_catalog(\n        db,\n        {\"id\": \"carol\"},\n        plugins,\n        VIEW_TABLE,\n        TABLE_CANDIDATES_SQL,\n        implicit_deny=True,\n    )\n    allowed_analytics = res_allowed(rows, parent=\"analytics\")\n    denied_analytics = res_denied(rows, parent=\"analytics\")\n\n    assert \"/analytics/secret\" in denied_analytics\n    # 10 analytics children total, 1 denied\n    assert len(allowed_analytics) == 9\n\n\n@pytest.mark.asyncio\nasync def test_specificity_child_allow_overrides_parent_deny_catalog(db):\n    await seed_catalog(db)\n    plugins = [\n        plugin_allow_all_for_user(\"alice\"),\n        plugin_org_policy_deny_parent(\"analytics\"),  # parent-level deny\n        plugin_child_allow_for_user(\n            \"alice\", \"analytics\", \"table02\"\n        ),  # child allow beats parent deny\n    ]\n    rows = await resolve_permissions_from_catalog(\n        db,\n        {\"id\": \"alice\"},\n        plugins,\n        VIEW_TABLE,\n        TABLE_CANDIDATES_SQL,\n        implicit_deny=True,\n    )\n\n    # table02 allowed, other analytics tables denied\n    assert any(r[\"resource\"] == \"/analytics/table02\" and r[\"allow\"] == 1 for r in rows)\n    assert all(\n        (r[\"parent\"] != \"analytics\" or r[\"child\"] == \"table02\" or r[\"allow\"] == 0)\n        for r in rows\n    )\n\n\n@pytest.mark.asyncio\nasync def test_root_deny_all_but_parent_allow_rescues_specific_parent_catalog(db):\n    await seed_catalog(db)\n    plugins = [\n        plugin_root_deny_for_all(),  # root deny\n        plugin_allow_parent_for_user(\n            \"bob\", \"accounting\"\n        ),  # parent allow (more specific)\n    ]\n    rows = await resolve_permissions_from_catalog(\n        db, {\"id\": \"bob\"}, plugins, VIEW_TABLE, TABLE_CANDIDATES_SQL, implicit_deny=True\n    )\n    for r in rows:\n        if r[\"parent\"] == \"accounting\":\n            assert r[\"allow\"] == 1\n        else:\n            assert r[\"allow\"] == 0\n\n\n@pytest.mark.asyncio\nasync def test_parent_scoped_candidates(db):\n    await seed_catalog(db)\n    plugins = [\n        plugin_org_policy_deny_parent(\"hr\"),\n        plugin_allow_parent_for_user(\"carol\", \"analytics\"),\n    ]\n    rows = await resolve_permissions_from_catalog(\n        db,\n        {\"id\": \"carol\"},\n        plugins,\n        VIEW_TABLE,\n        PARENT_CANDIDATES_SQL,\n        implicit_deny=True,\n    )\n    d = {r[\"resource\"]: r[\"allow\"] for r in rows}\n    assert d[\"/analytics\"] == 1\n    assert d[\"/hr\"] == 0\n\n\n@pytest.mark.asyncio\nasync def test_implicit_deny_behavior(db):\n    await seed_catalog(db)\n    plugins = []  # no rules at all\n\n    # implicit_deny=True -> everything denied with reason 'implicit deny'\n    rows = await resolve_permissions_from_catalog(\n        db,\n        {\"id\": \"erin\"},\n        plugins,\n        VIEW_TABLE,\n        TABLE_CANDIDATES_SQL,\n        implicit_deny=True,\n    )\n    assert all(r[\"allow\"] == 0 and r[\"reason\"] == \"implicit deny\" for r in rows)\n\n    # implicit_deny=False -> no winner => allow is None, reason is None\n    rows2 = await resolve_permissions_from_catalog(\n        db,\n        {\"id\": \"erin\"},\n        plugins,\n        VIEW_TABLE,\n        TABLE_CANDIDATES_SQL,\n        implicit_deny=False,\n    )\n    assert all(r[\"allow\"] is None and r[\"reason\"] is None for r in rows2)\n\n\n@pytest.mark.asyncio\nasync def test_candidate_filters_via_params(db):\n    await seed_catalog(db)\n    # Add some metadata to test filtering\n    # Mark 'hr' as is_memory=1 and increment analytics schema_version\n    await db.execute_write(\n        \"UPDATE catalog_databases SET is_memory=1 WHERE database_name='hr'\"\n    )\n    await db.execute_write(\n        \"UPDATE catalog_databases SET schema_version=2 WHERE database_name='analytics'\"\n    )\n\n    # Candidate SQL that filters by db metadata via params\n    candidate_sql = \"\"\"\n    SELECT t.database_name AS parent, t.table_name AS child\n    FROM catalog_tables t\n    JOIN catalog_databases d ON d.database_name = t.database_name\n    WHERE (:exclude_memory = 1 AND d.is_memory = 1) IS NOT 1\n      AND (:min_schema_version IS NULL OR d.schema_version >= :min_schema_version)\n    \"\"\"\n\n    plugins = [\n        plugin_root_deny_for_all(),\n        plugin_allow_parent_for_user(\n            \"dev\", \"analytics\"\n        ),  # analytics rescued if included by candidates\n    ]\n\n    # Case 1: exclude memory dbs, require schema_version >= 2 -> only analytics appear, and thus are allowed\n    rows = await resolve_permissions_from_catalog(\n        db,\n        {\"id\": \"dev\"},\n        plugins,\n        VIEW_TABLE,\n        candidate_sql,\n        candidate_params={\"exclude_memory\": 1, \"min_schema_version\": 2},\n        implicit_deny=True,\n    )\n    assert rows and all(r[\"parent\"] == \"analytics\" for r in rows)\n    assert all(r[\"allow\"] == 1 for r in rows)\n\n    # Case 2: include memory dbs, min_schema_version = None -> accounting/hr/analytics appear,\n    # but root deny wins except where specifically allowed (none except analytics parent allow doesn’t apply to table depth if candidate includes children; still fine—policy is explicit).\n    rows2 = await resolve_permissions_from_catalog(\n        db,\n        {\"id\": \"dev\"},\n        plugins,\n        VIEW_TABLE,\n        candidate_sql,\n        candidate_params={\"exclude_memory\": 0, \"min_schema_version\": None},\n        implicit_deny=True,\n    )\n    assert any(r[\"parent\"] == \"accounting\" for r in rows2)\n    assert any(r[\"parent\"] == \"hr\" for r in rows2)\n    # For table-scoped candidates, the parent-level allow does not override root deny unless you have child-level rules\n    assert all(r[\"allow\"] in (0, 1) for r in rows2)\n\n\n@pytest.mark.asyncio\nasync def test_action_specific_rules(db):\n    await seed_catalog(db)\n    plugins = [plugin_allow_all_for_action(\"dana\", VIEW_TABLE)]\n\n    view_rows = await resolve_permissions_from_catalog(\n        db,\n        {\"id\": \"dana\"},\n        plugins,\n        VIEW_TABLE,\n        TABLE_CANDIDATES_SQL,\n        implicit_deny=True,\n    )\n    assert view_rows and all(r[\"allow\"] == 1 for r in view_rows)\n    assert all(r[\"action\"] == VIEW_TABLE for r in view_rows)\n\n    insert_rows = await resolve_permissions_from_catalog(\n        db,\n        {\"id\": \"dana\"},\n        plugins,\n        \"insert-row\",\n        TABLE_CANDIDATES_SQL,\n        implicit_deny=True,\n    )\n    assert insert_rows and all(r[\"allow\"] == 0 for r in insert_rows)\n    assert all(r[\"reason\"] == \"implicit deny\" for r in insert_rows)\n    assert all(r[\"action\"] == \"insert-row\" for r in insert_rows)\n\n\n@pytest.mark.asyncio\nasync def test_actor_actor_id_action_parameters_available(db):\n    \"\"\"Test that :actor (JSON), :actor_id, and :action are all available in SQL\"\"\"\n    await seed_catalog(db)\n\n    def plugin_using_all_parameters() -> Callable[[str], PermissionSQL]:\n        def provider(action: str) -> PermissionSQL:\n            return PermissionSQL(\"\"\"\n                SELECT NULL AS parent, NULL AS child, 1 AS allow,\n                       'Actor ID: ' || COALESCE(:actor_id, 'null') ||\n                       ', Actor JSON: ' || COALESCE(:actor, 'null') ||\n                       ', Action: ' || :action AS reason\n                WHERE :actor_id = 'test_user' AND :action = 'view-table'\n                AND json_extract(:actor, '$.role') = 'admin'\n                \"\"\")\n\n        return provider\n\n    plugins = [plugin_using_all_parameters()]\n\n    # Test with full actor dict\n    rows = await resolve_permissions_from_catalog(\n        db,\n        {\"id\": \"test_user\", \"role\": \"admin\"},\n        plugins,\n        \"view-table\",\n        TABLE_CANDIDATES_SQL,\n        implicit_deny=True,\n    )\n\n    # Should have allowed rows with reason containing all the info\n    allowed = [r for r in rows if r[\"allow\"] == 1]\n    assert len(allowed) > 0\n\n    # Check that the reason string contains evidence of all parameters\n    reason = allowed[0][\"reason\"]\n    assert \"test_user\" in reason\n    assert \"view-table\" in reason\n    # The :actor parameter should be the JSON string\n    assert \"Actor JSON:\" in reason\n\n\n@pytest.mark.asyncio\nasync def test_multiple_plugins_with_own_parameters(db):\n    \"\"\"\n    Test that multiple plugins can use their own parameter names without conflict.\n\n    This verifies that the parameter naming convention works: plugins prefix their\n    parameters (e.g., :plugin1_pattern, :plugin2_message) and both sets of parameters\n    are successfully bound in the SQL queries.\n    \"\"\"\n    await seed_catalog(db)\n\n    def plugin_one() -> Callable[[str], PermissionSQL]:\n        def provider(action: str) -> PermissionSQL:\n            if action != \"view-table\":\n                return PermissionSQL(\"plugin_one\", \"SELECT NULL WHERE 0\", {})\n            return PermissionSQL(\n                \"\"\"\n                SELECT database_name AS parent, table_name AS child,\n                       1 AS allow, 'Plugin one used param: ' || :plugin1_param AS reason\n                FROM catalog_tables\n                WHERE database_name = 'accounting'\n                \"\"\",\n                {\n                    \"plugin1_param\": \"value1\",\n                },\n            )\n\n        return provider\n\n    def plugin_two() -> Callable[[str], PermissionSQL]:\n        def provider(action: str) -> PermissionSQL:\n            if action != \"view-table\":\n                return PermissionSQL(\"plugin_two\", \"SELECT NULL WHERE 0\", {})\n            return PermissionSQL(\n                \"\"\"\n                SELECT database_name AS parent, table_name AS child,\n                       1 AS allow, 'Plugin two used param: ' || :plugin2_param AS reason\n                FROM catalog_tables\n                WHERE database_name = 'hr'\n                \"\"\",\n                {\n                    \"plugin2_param\": \"value2\",\n                },\n            )\n\n        return provider\n\n    plugins = [plugin_one(), plugin_two()]\n\n    rows = await resolve_permissions_from_catalog(\n        db,\n        {\"id\": \"test_user\"},\n        plugins,\n        \"view-table\",\n        TABLE_CANDIDATES_SQL,\n        implicit_deny=False,\n    )\n\n    # Both plugins should contribute results with their parameters successfully bound\n    plugin_one_rows = [\n        r for r in rows if r.get(\"reason\") and \"Plugin one\" in r[\"reason\"]\n    ]\n    plugin_two_rows = [\n        r for r in rows if r.get(\"reason\") and \"Plugin two\" in r[\"reason\"]\n    ]\n\n    assert len(plugin_one_rows) > 0, \"Plugin one should contribute rules\"\n    assert len(plugin_two_rows) > 0, \"Plugin two should contribute rules\"\n\n    # Verify each plugin's parameters were successfully bound in the SQL\n    assert any(\n        \"value1\" in r.get(\"reason\", \"\") for r in plugin_one_rows\n    ), \"Plugin one's :plugin1_param should be bound\"\n    assert any(\n        \"value2\" in r.get(\"reason\", \"\") for r in plugin_two_rows\n    ), \"Plugin two's :plugin2_param should be bound\"\n"
  },
  {
    "path": "tests/test_write_wrapper.py",
    "content": "\"\"\"\nTests for the write_wrapper plugin hook.\n\"\"\"\n\nfrom datasette.app import Datasette\nfrom datasette.hookspecs import hookimpl\nfrom datasette.plugins import pm\nimport pytest\nimport sqlite3\nimport time\n\n\n@pytest.fixture\ndef datasette(tmp_path):\n    db_path = str(tmp_path / \"test.db\")\n    ds = Datasette([db_path])\n    return ds\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"use_execute_write\",\n    (False, True),\n    ids=[\"execute_write_fn\", \"execute_write\"],\n)\nasync def test_write_wrapper_before_and_after(datasette, use_execute_write):\n    \"\"\"Test that code before and after yield both execute.\"\"\"\n    log = []\n\n    class Plugin:\n        __name__ = \"Plugin\"\n\n        @staticmethod\n        @hookimpl\n        def write_wrapper(datasette, database, request, transaction):\n            def wrapper(conn):\n                log.append(\"before\")\n                yield\n                log.append(\"after\")\n\n            return wrapper\n\n    pm.register(Plugin(), name=\"test_before_after\")\n    try:\n        db = datasette.get_database(\"test\")\n        if use_execute_write:\n            await db.execute_write(\n                \"create table if not exists t (id integer primary key)\"\n            )\n        else:\n            await db.execute_write_fn(\n                lambda conn: conn.execute(\n                    \"create table if not exists t (id integer primary key)\"\n                )\n            )\n        assert log == [\"before\", \"after\"]\n    finally:\n        pm.unregister(name=\"test_before_after\")\n\n\n@pytest.mark.asyncio\nasync def test_write_wrapper_receives_result_via_yield(datasette):\n    \"\"\"Test that the result of fn(conn) is sent back through yield.\"\"\"\n    captured = {}\n\n    class Plugin:\n        __name__ = \"Plugin\"\n\n        @staticmethod\n        @hookimpl\n        def write_wrapper(datasette, database, request, transaction):\n            def wrapper(conn):\n                result = yield\n                captured[\"result\"] = result\n\n            return wrapper\n\n    pm.register(Plugin(), name=\"test_result\")\n    try:\n        db = datasette.get_database(\"test\")\n        await db.execute_write_fn(\n            lambda conn: conn.execute(\n                \"create table if not exists t2 (id integer primary key)\"\n            )\n        )\n        assert \"result\" in captured\n        # Should be a sqlite3 Cursor\n        assert captured[\"result\"] is not None\n    finally:\n        pm.unregister(name=\"test_result\")\n\n\n@pytest.mark.asyncio\nasync def test_write_wrapper_exception_thrown_into_generator(datasette):\n    \"\"\"Test that exceptions from fn(conn) are thrown into the generator.\"\"\"\n    caught = {}\n\n    class Plugin:\n        __name__ = \"Plugin\"\n\n        @staticmethod\n        @hookimpl\n        def write_wrapper(datasette, database, request, transaction):\n            def wrapper(conn):\n                try:\n                    yield\n                except Exception as e:\n                    caught[\"error\"] = e\n\n            return wrapper\n\n    pm.register(Plugin(), name=\"test_exception\")\n    try:\n        db = datasette.get_database(\"test\")\n        with pytest.raises(Exception, match=\"deliberate\"):\n            await db.execute_write_fn(\n                lambda conn: (_ for _ in ()).throw(Exception(\"deliberate\"))\n            )\n        assert \"error\" in caught\n        assert str(caught[\"error\"]) == \"deliberate\"\n    finally:\n        pm.unregister(name=\"test_exception\")\n\n\n@pytest.mark.asyncio\nasync def test_write_wrapper_conn_is_usable(datasette):\n    \"\"\"Test that the conn passed to the wrapper can execute SQL.\"\"\"\n\n    class Plugin:\n        __name__ = \"Plugin\"\n\n        @staticmethod\n        @hookimpl\n        def write_wrapper(datasette, database, request, transaction):\n            def wrapper(conn):\n                conn.execute(\"create table if not exists hook_log (msg text)\")\n                conn.execute(\"insert into hook_log values ('before')\")\n                yield\n                conn.execute(\"insert into hook_log values ('after')\")\n\n            return wrapper\n\n    pm.register(Plugin(), name=\"test_conn\")\n    try:\n        db = datasette.get_database(\"test\")\n        await db.execute_write_fn(\n            lambda conn: conn.execute(\n                \"create table if not exists t3 (id integer primary key)\"\n            )\n        )\n        result = await db.execute(\"select msg from hook_log order by rowid\")\n        messages = [row[0] for row in result.rows]\n        assert messages == [\"before\", \"after\"]\n    finally:\n        pm.unregister(name=\"test_conn\")\n\n\n@pytest.mark.asyncio\nasync def test_write_wrapper_multiple_plugins_nest(datasette):\n    \"\"\"Test that multiple write_wrapper plugins nest correctly.\"\"\"\n    log = []\n\n    class PluginA:\n        __name__ = \"PluginA\"\n\n        @staticmethod\n        @hookimpl\n        def write_wrapper(datasette, database, request, transaction):\n            def wrapper(conn):\n                log.append(\"A-before\")\n                yield\n                log.append(\"A-after\")\n\n            return wrapper\n\n    class PluginB:\n        __name__ = \"PluginB\"\n\n        @staticmethod\n        @hookimpl\n        def write_wrapper(datasette, database, request, transaction):\n            def wrapper(conn):\n                log.append(\"B-before\")\n                yield\n                log.append(\"B-after\")\n\n            return wrapper\n\n    pm.register(PluginA(), name=\"PluginA\")\n    pm.register(PluginB(), name=\"PluginB\")\n    try:\n        db = datasette.get_database(\"test\")\n        await db.execute_write_fn(\n            lambda conn: conn.execute(\n                \"create table if not exists t4 (id integer primary key)\"\n            )\n        )\n        assert set(log) == {\"A-before\", \"A-after\", \"B-before\", \"B-after\"}\n        # Verify proper nesting: each plugin's before/after should be\n        # symmetric around the write\n        a_before = log.index(\"A-before\")\n        a_after = log.index(\"A-after\")\n        b_before = log.index(\"B-before\")\n        b_after = log.index(\"B-after\")\n        if a_before < b_before:\n            assert a_after > b_after, \"A is outer so A-after should come after B-after\"\n        else:\n            assert b_after > a_after, \"B is outer so B-after should come after A-after\"\n    finally:\n        pm.unregister(name=\"PluginA\")\n        pm.unregister(name=\"PluginB\")\n\n\n@pytest.mark.asyncio\nasync def test_write_wrapper_return_none_skips(datasette):\n    \"\"\"Test that returning None from write_wrapper means no wrapping.\"\"\"\n    log = []\n\n    class Plugin:\n        __name__ = \"Plugin\"\n\n        @staticmethod\n        @hookimpl\n        def write_wrapper(datasette, database, request, transaction):\n            log.append(\"hook-called\")\n            return None\n\n    pm.register(Plugin(), name=\"test_skip\")\n    try:\n        db = datasette.get_database(\"test\")\n        await db.execute_write_fn(\n            lambda conn: conn.execute(\n                \"create table if not exists t5 (id integer primary key)\"\n            )\n        )\n        assert log == [\"hook-called\"]\n    finally:\n        pm.unregister(name=\"test_skip\")\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"request_value,transaction_value,expected_request,expected_transaction\",\n    (\n        (\"fake-request\", True, \"fake-request\", True),\n        (None, True, None, True),\n        (None, False, None, False),\n    ),\n    ids=[\"with-request\", \"request-none-by-default\", \"transaction-false\"],\n)\nasync def test_write_wrapper_hook_parameters(\n    datasette,\n    request_value,\n    transaction_value,\n    expected_request,\n    expected_transaction,\n):\n    \"\"\"Test that request and transaction parameters are passed through.\"\"\"\n    captured = {}\n\n    class Plugin:\n        __name__ = \"Plugin\"\n\n        @staticmethod\n        @hookimpl\n        def write_wrapper(datasette, database, request, transaction):\n            captured[\"request\"] = request\n            captured[\"database\"] = database\n            captured[\"transaction\"] = transaction\n\n    pm.register(Plugin(), name=\"test_params\")\n    try:\n        db = datasette.get_database(\"test\")\n        kwargs = {\"transaction\": transaction_value}\n        if request_value is not None:\n            kwargs[\"request\"] = request_value\n        await db.execute_write_fn(\n            lambda conn: conn.execute(\n                \"create table if not exists t6 (id integer primary key)\"\n            ),\n            **kwargs,\n        )\n        assert captured[\"request\"] == expected_request\n        assert captured[\"database\"] == \"test\"\n        assert captured[\"transaction\"] == expected_transaction\n    finally:\n        pm.unregister(name=\"test_params\")\n\n\n@pytest.mark.asyncio\nasync def test_write_wrapper_via_api(tmp_path):\n    \"\"\"Test that write_wrapper fires for API write operations.\"\"\"\n    log = []\n\n    db_path = str(tmp_path / \"test.db\")\n    ds = Datasette([db_path], pdb=False)\n    ds.root_enabled = True\n\n    class Plugin:\n        __name__ = \"Plugin\"\n\n        @staticmethod\n        @hookimpl\n        def write_wrapper(datasette, database, request, transaction):\n            if database != \"test\":\n                return None\n\n            def wrapper(conn):\n                log.append(\"before\")\n                yield\n                log.append(\"after\")\n\n            return wrapper\n\n    pm.register(Plugin(), name=\"test_api\")\n    try:\n        db = ds.get_database(\"test\")\n        await db.execute_write(\n            \"create table if not exists api_test (id integer primary key, name text)\"\n        )\n        log.clear()\n\n        token = \"dstok_{}\".format(\n            ds.sign(\n                {\"a\": \"root\", \"token\": \"dstok\", \"t\": int(time.time())},\n                namespace=\"token\",\n            )\n        )\n        response = await ds.client.post(\n            \"/test/api_test/-/insert\",\n            json={\"row\": {\"name\": \"test\"}, \"return\": True},\n            headers={\n                \"Authorization\": \"Bearer {}\".format(token),\n                \"Content-Type\": \"application/json\",\n            },\n        )\n        assert response.status_code == 201, response.json()\n        assert log == [\"before\", \"after\"]\n    finally:\n        pm.unregister(name=\"test_api\")\n\n\n@pytest.mark.asyncio\nasync def test_write_wrapper_change_group_pattern(datasette):\n    \"\"\"Test the motivating use case: activating a change group around a write.\"\"\"\n    db = datasette.get_database(\"test\")\n\n    await db.execute_write(\n        \"create table if not exists groups (id integer primary key, current integer)\"\n    )\n    await db.execute_write(\n        \"create table if not exists data (id integer primary key, value text)\"\n    )\n    await db.execute_write(\"insert into groups (id, current) values (1, null)\")\n\n    class Plugin:\n        __name__ = \"Plugin\"\n\n        @staticmethod\n        @hookimpl\n        def write_wrapper(datasette, database, request, transaction):\n            if request and getattr(request, \"group_id\", None):\n                group_id = request.group_id\n\n                def wrapper(conn):\n                    conn.execute(\n                        \"update groups set current = 1 where id = ?\", [group_id]\n                    )\n                    yield\n                    conn.execute(\"update groups set current = null where current = 1\")\n\n                return wrapper\n\n    pm.register(Plugin(), name=\"test_change_group\")\n    try:\n\n        class FakeRequest:\n            group_id = 1\n\n        await db.execute_write_fn(\n            lambda conn: conn.execute(\"insert into data (value) values ('test')\"),\n            request=FakeRequest(),\n        )\n\n        result = await db.execute(\"select current from groups where id = 1\")\n        assert result.rows[0][0] is None\n    finally:\n        pm.unregister(name=\"test_change_group\")\n\n\nWRITE_ACTIONS = (\n    sqlite3.SQLITE_INSERT,\n    sqlite3.SQLITE_UPDATE,\n    sqlite3.SQLITE_DELETE,\n)\n\n\n@pytest.mark.asyncio\n@pytest.mark.parametrize(\n    \"actor,table,should_deny\",\n    (\n        (None, \"protected_table\", True),\n        ({\"id\": \"regular\"}, \"protected_table\", True),\n        ({\"id\": \"admin\"}, \"protected_table\", False),\n        (None, \"other_table\", False),\n        ({\"id\": \"regular\"}, \"other_table\", False),\n    ),\n    ids=[\n        \"no-actor-protected\",\n        \"regular-user-protected\",\n        \"admin-protected\",\n        \"no-actor-other\",\n        \"regular-user-other\",\n    ],\n)\nasync def test_write_wrapper_set_authorizer(datasette, actor, table, should_deny):\n    \"\"\"Test the docs example that uses set_authorizer to block writes to a protected table.\"\"\"\n    db = datasette.get_database(\"test\")\n    await db.execute_write(\n        \"create table if not exists protected_table (id integer primary key, value text)\"\n    )\n    await db.execute_write(\n        \"create table if not exists other_table (id integer primary key, value text)\"\n    )\n\n    class Plugin:\n        __name__ = \"Plugin\"\n\n        @staticmethod\n        @hookimpl\n        def write_wrapper(datasette, database, request, transaction):\n            actor = None\n            if request:\n                actor = request.actor\n            if actor and actor.get(\"id\") == \"admin\":\n                return None\n\n            def wrapper(conn):\n                def authorizer(action, arg1, arg2, db_name, trigger):\n                    if action in WRITE_ACTIONS and arg1 == \"protected_table\":\n                        return sqlite3.SQLITE_DENY\n                    return sqlite3.SQLITE_OK\n\n                conn.set_authorizer(authorizer)\n                try:\n                    yield\n                finally:\n                    conn.set_authorizer(lambda *args: sqlite3.SQLITE_OK)\n\n            return wrapper\n\n    class FakeRequest:\n        def __init__(self, actor):\n            self.actor = actor\n\n    pm.register(Plugin(), name=\"test_set_authorizer\")\n    try:\n        request = FakeRequest(actor)\n        if should_deny:\n            with pytest.raises(Exception):\n                await db.execute_write_fn(\n                    lambda conn: conn.execute(\n                        f\"insert into {table} (value) values ('test')\"\n                    ),\n                    request=request,\n                )\n        else:\n            await db.execute_write_fn(\n                lambda conn: conn.execute(\n                    f\"insert into {table} (value) values ('test')\"\n                ),\n                request=request,\n            )\n            result = await db.execute(\n                f\"select value from {table} order by rowid desc limit 1\"\n            )\n            assert result.rows[0][0] == \"test\"\n    finally:\n        pm.unregister(name=\"test_set_authorizer\")\n"
  },
  {
    "path": "tests/utils.py",
    "content": "from datasette.utils.sqlite import sqlite3\n\n\ndef last_event(datasette):\n    events = getattr(datasette, \"_tracked_events\", [])\n    return events[-1] if events else None\n\n\ndef assert_footer_links(soup):\n    footer_links = soup.find(\"footer\").find_all(\"a\")\n    assert 4 == len(footer_links)\n    datasette_link, license_link, source_link, about_link = footer_links\n    assert \"Datasette\" == datasette_link.text.strip()\n    assert \"tests/fixtures.py\" == source_link.text.strip()\n    assert \"Apache License 2.0\" == license_link.text.strip()\n    assert \"About Datasette\" == about_link.text.strip()\n    assert \"https://datasette.io/\" == datasette_link[\"href\"]\n    assert (\n        \"https://github.com/simonw/datasette/blob/main/tests/fixtures.py\"\n        == source_link[\"href\"]\n    )\n    assert (\n        \"https://github.com/simonw/datasette/blob/main/LICENSE\" == license_link[\"href\"]\n    )\n    assert \"https://github.com/simonw/datasette\" == about_link[\"href\"]\n\n\ndef inner_html(soup):\n    html = str(soup)\n    # This includes the parent tag - so remove that\n    inner_html = html.split(\">\", 1)[1].rsplit(\"<\", 1)[0]\n    return inner_html.strip()\n\n\ndef has_load_extension():\n    conn = sqlite3.connect(\":memory:\")\n    return hasattr(conn, \"enable_load_extension\")\n\n\ndef cookie_was_deleted(response, cookie):\n    return any(\n        h\n        for h in response.headers.get_list(\"set-cookie\")\n        if h.startswith(f'{cookie}=\"\";')\n    )\n"
  }
]
    this …http…1hello11a1b1c1abca/b.c-dc\n \n  https://twitter.com/simonw\n \n<Binary:\\xa07\\xa0bytes>\\xa0http…http…\\xa0http…1hello2world3\\xa0{}{i}{i}a{i}b{i}c{i}a,babca/b,.c-da/b.c-dcd,ede{\"row\": {\"pk1\": \"d\", \"pk2\": \"e\", \"content\": \"RENDER_CELL_DEMO\"}, \"column\": \"content\", \"table\": \"compound_primary_key\", \"database\": \"fixtures\", \"pks\": [\"pk1\", \"pk2\"], \"config\": {\"depth\": \"database\"}}1hello\\xa01-\\xa031ab2\\xa0\\xa0\\xa0\\xa0\\xa01131ab1world2\\xa011,feline1\\xa01feline2,canine2\\xa02caninehelloHELLOworldWORLD\\xa0\\xa011<Binary:\\xa07\\xa0bytes>22<Binary:\\xa07\\xa0bytes>33\\xa03Detroit2Los Angeles4Memnonia1San Francisco2Paranormal1Museum